Azure blob logger in ASP.NET Core

This post shows you an implementation of a blob logger in ASP.NET Core. This logger stores logs as Append Blobs in an Azure blob storage account. Logs can be retreived while they are created, you can make sure that new logs not overwrite stored logs and you can delete old logs.

Blobs are stored in a container in your storage account, you can group logs inside a container by adding one or more “foldername/” before the file name. You can for example set the blob name to “201902/log-20190201.log”.

Options

Our blob logger needs a connection string and a container name. The option class is shown below.

public class BlobStorageOptions
{
    #region Variables

    public string ConnectionString { get; set; }
    public string ContainerName { get; set; }

    #endregion

    #region Constructors

    public BlobStorageOptions()
    {
        // Set values for instance variables
        this.ConnectionString = null;
        this.ContainerName = null;

    } // End of the constructor

    #endregion

} // End of the class

Model

This model is returned in a list in one of the methods in the blob logger class.

public class BlobLog
{
    #region Variables

    public string log_date { get; set; } // yyyy-MM-ddThh:mm:ss
    public string log_name { get; set; }

    #endregion

    #region Constructors

    public BlobLog()
    {
        // Set values for instance variables
        this.log_date = null;
        this.log_name = null;

    } // End of the constructor

    public BlobLog(string log_date, string log_name)
    {
        // Set values for instance variables
        this.log_date = log_date;
        this.log_name = log_name;

    } // End of the constructor

    #endregion

} // End of the class

Interface

This interface shows all the methods that need to be implemented by our blob logger class.

public interface IBlobLogger
{
    Task LogDebug(string blob_name, string message);
    Task LogInformation(string blob_name, string message);
    Task LogWarning(string blob_name, string message);
    Task LogError(string blob_name, string message);
    Task GetLogAsStream(string blob_name, Stream stream);
    Task<string> GetLogAsString(string blob_name);
    IList<BlobLog> GetBlobLogs(string email);
    Task<bool> LogExists(string blob_name);
    Task Delete(string blob_name);
    Task DeleteByLastModifiedDate(Int32 days);

} // End of the interface

Class

This is the blob logger class. The last method (DeleteByLastModifiedDate) is used to clean up older logs but it might be better to use a life cycle management policy in Azure to delete older blobs.

public class BlobLogger : IBlobLogger
{
    #region Variables

    private readonly BlobStorageOptions options;
    private readonly CloudBlobClient client;
    private readonly CloudBlobContainer container;

    #endregion

    #region Constructors

    public BlobLogger(IOptions<BlobStorageOptions> options)
    {
        // Set values for instance variables
        this.options = options.Value;

        // Get a storage account
        CloudStorageAccount account = CloudStorageAccount.Parse(this.options.ConnectionString);

        // Get a client
        this.client = account.CreateCloudBlobClient();

        // Get a container reference
        this.container = this.client.GetContainerReference(this.options.ContainerName);

        // Create a container if it doesn't exist
        this.container.CreateIfNotExists();

    } // End of the constructor

    #endregion

    #region Log methods

    public async Task LogDebug(string blob_name, string message)
    {
        if (string.IsNullOrEmpty(message) == false)
        {
            await WriteToAppendBlob(blob_name, $"{DateTime.UtcNow.ToString("o")} [DBG] {message}" + Environment.NewLine);
        }

    } // End of the LogDebug method

    public async Task LogInformation(string blob_name, string message)
    {
        if (string.IsNullOrEmpty(message) == false)
        {
            await WriteToAppendBlob(blob_name, $"{DateTime.UtcNow.ToString("o")} [INF] {message}" + Environment.NewLine);
        }

    } // End of the LogInformation method

    public async Task LogWarning(string blob_name, string message)
    {
        if (string.IsNullOrEmpty(message) == false)
        {
            await WriteToAppendBlob(blob_name, $"{DateTime.UtcNow.ToString("o")} [WRN] {message}" + Environment.NewLine);
        }

    } // End of the LogWarning method

    public async Task LogError(string blob_name, string message)
    {
        if(string.IsNullOrEmpty(message) == false)
        {
            await WriteToAppendBlob(blob_name, $"{DateTime.UtcNow.ToString("o")} [ERR] {message}" + Environment.NewLine);
        }

    } // End of the LogError method

    #endregion

    #region Get methods

    public async Task GetLogAsStream(string blob_name, Stream stream)
    {
        // Get a blob object
        CloudAppendBlob blob = this.container.GetAppendBlobReference(blob_name);

        // Download the blob to a stream
        await blob.DownloadToStreamAsync(stream);

    } // End of the GetLogAsStream method

    public async Task<string> GetLogAsString(string blob_name)
    {
        // Create a string to return
        string log = "";

        // Get a blob object
        CloudAppendBlob blob = this.container.GetAppendBlobReference(blob_name);

        // Get the append blob
        using (MemoryStream stream = new MemoryStream())
        {
            await blob.DownloadToStreamAsync(stream);
            stream.Seek(0, SeekOrigin.Begin);
            log = Encoding.UTF8.GetString(stream.ToArray());
        }

        // Return the log
        return log;

    } // End of the GetLogAsString method

    public IList<BlobLog> GetBlobLogs(string email)
    {
        // Create the variable to return
        IList<BlobLog> logs = new List<BlobLog>();

        // Get a list with blobs
        IEnumerable<IListBlobItem> blobs = this.container.ListBlobs(email + "/", true, BlobListingDetails.None);

        foreach (IListBlobItem item in blobs)
        {
            // Get a blob reference
            CloudBlob blob = (CloudBlob)item;

            // Add the blob log
            logs.Add(new BlobLog(blob.Properties.LastModified.Value.ToString("yyyy-MM-ddThh:mm:ss"), blob.Name));
        }

        // Return logs
        return logs;

    } // End of the GetBlobLogs method

    #endregion

    #region Property methods

    public async Task<bool> LogExists(string blob_name)
    {
        // Get a blob reference
        CloudBlob blob = this.container.GetBlobReference(blob_name);

        // Return a boolean
        return await blob.ExistsAsync();

    } // End of the LogExists method

    #endregion

    #region Delete methods

    public async Task Delete(string blob_name)
    {
        // Get a blob object
        CloudBlob blob = this.container.GetBlobReference(blob_name);

        // Delete blob
        await blob.DeleteIfExistsAsync();
            
    } // End of the Delete method

    public async Task DeleteByLastModifiedDate(Int32 days)
    {
        // Get a list with blobs
        BlobResultSegment blob_segment = await this.container.ListBlobsSegmentedAsync("", true, BlobListingDetails.All, 100, null, null, null);
 
        // Set the date treshold
        DateTime treshold = DateTime.UtcNow.AddDays(days * -1);

        // Create an endless loop
        while(true)
        {
            // Delete blobs
            foreach (IListBlobItem item in blob_segment.Results)
            {
                // Get a blob reference
                CloudBlob blob = (CloudBlob)item;

                // Delete a blob if it is older than the threshold
                if(blob.Properties.LastModified.Value.UtcDateTime < treshold)
                {
                    await blob.DeleteIfExistsAsync();
                }
            }

            // Check if more blobs can be found
            if(blob_segment.ContinuationToken != null)
            {
                blob_segment = await this.container.ListBlobsSegmentedAsync("", true, BlobListingDetails.All, 100, blob_segment.ContinuationToken, null, null);
            }
            else
            {
                break;
            }
        }

    } // End of the DeleteByLastModifiedDate method

    #endregion

    #region Helper methods

    private async Task WriteToAppendBlob(string blob_name, string log)
    {
        // Get a blob reference
        CloudAppendBlob blob = this.container.GetAppendBlobReference(blob_name);

        // Create a blob if it doesn't exist
        if (await blob.ExistsAsync() == false)
        {
            await blob.CreateOrReplaceAsync();
        }

        // Append the log to a blob
        using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(log)))
        {
            // Append text to the blob
            await blob.AppendBlockAsync(stream);
        }

    } // End of the WriteToAppendBlob method

    #endregion

} // End of the class

Life Cycle Management

The policy below can be used to delete older blobs, the policy deletes blobs older than 30 days in container fortnox-logs and in container test-fortnox-logs.

{
    "rules": [
        {
            "enabled": true,
            "name": "delete-old-blobs",
            "type": "Lifecycle",
            "definition": {
                "actions": {
                    "baseBlob": {
                        "delete": {
                            "daysAfterModificationGreaterThan": 30
                        }
                    }
                },
                "filters": {
                    "blobTypes": [
                        "blockBlob",
                        "appendBlob"
                    ],
                    "prefixMatch": [
                        "fortnox-logs",
                        "test-fortnox-logs"
                    ]
                }
            }
        }
    ]
}

Leave a Reply

Your email address will not be published. Required fields are marked *