Skip to content

Instantly share code, notes, and snippets.

@ccpu

ccpu/AutoMigration.cs

Forked from lakeman/AutoMigration.cs
Created November 25, 2020 13:37
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ccpu/e13a935592ecaac9fb55ddf9ed4669d0 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ccpu/e13a935592ecaac9fb55ddf9ed4669d0 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @lakeman lakeman revised this gist Dec 10, 2019. 1 changed file with 51 additions and 25 deletions.
    76 changes: 51 additions & 25 deletions AutoMigration.cs
    View file
    Original file line number Diff line number Diff line change
    @@ -29,13 +29,22 @@ public class AutoMigration : IOperationReporter
    private readonly DbContext db;
    private readonly ILogger logger;

    public enum MigrationResult
    {
    Noop,
    Created,
    Migrated,
    AutoMigrated
    }

    public AutoMigration(DbContext db, ILogger logger)
    {
    this.db = db;
    this.logger = logger;
    }

    public bool AllowDestructive { get; set; } = false;
    public bool AllowDestructive { get; set; } = true;
    public bool MigrateNewDatabase { get; set; } = false;

    void IOperationReporter.WriteError(string message) => logger.LogError(message);
    void IOperationReporter.WriteInformation(string message) => logger.LogInformation(message);
    @@ -67,6 +76,10 @@ private async Task<string> ReadSnapshotSource()
    using var stream = new GZipStream(reader.GetStream(0), CompressionMode.Decompress);
    return await new StreamReader(stream).ReadToEndAsync();
    }
    catch (Exception)
    {
    return null;
    }
    finally
    {
    await db.Database.CloseConnectionAsync();
    @@ -141,19 +154,31 @@ private async Task<string> AutoMigrate(Assembly migrationAssembly, IModel oldMod
    var builder = new DesignTimeServicesBuilder(migrationAssembly, Assembly.GetEntryAssembly(), this, null);
    var services = builder.Build(db);
    var dependencies = services.GetRequiredService<MigrationsScaffolderDependencies>();
    var name = $"{DateTime.Now:yyyyMMddHHmmss}_Auto";
    var name = dependencies.MigrationsIdGenerator.GenerateId("Auto");

    // insert an extra step to track the history of auto migrations
    var insert = dependencies.HistoryRepository.GetInsertScript(
    new HistoryRow(
    name,
    (string)newModel.FindAnnotation("ProductVersion")?.Value ?? "Unknown version"
    ));

    if (oldModel == null)
    {
    //(not tested...)
    await db.Database.EnsureCreatedAsync();
    await db.Database.ExecuteSqlRawAsync(dependencies.HistoryRepository.GetCreateScript());
    await db.Database.ExecuteSqlRawAsync(insert);
    }
    else
    {
    // apply fixes for upgrading between major / minor versions
    oldModel = dependencies.SnapshotModelProcessor.Process(oldModel);

    var operations = dependencies.MigrationsModelDiffer.GetDifferences(oldModel, newModel);
    var operations = dependencies.MigrationsModelDiffer
    .GetDifferences(oldModel, newModel)
    // Ignore all seed updates. Workaround for (https://github.com/aspnet/EntityFrameworkCore/issues/18943)
    .Where(o => !(o is UpdateDataOperation))
    .ToList();

    if (!operations.Any())
    return null;
    @@ -162,15 +187,10 @@ private async Task<string> AutoMigrate(Assembly migrationAssembly, IModel oldMod
    throw new InvalidOperationException(
    "Automatic migration was not applied because it could result in data loss.");

    // insert an extra step to track the history of auto migrations
    var insert = new InsertDataOperation()
    {
    Table = "__EFMigrationsHistory", // TODO the table name can be overridden in config
    Schema = "dbo",
    Columns = new[] { nameof(HistoryRow.MigrationId), nameof(HistoryRow.ProductVersion) },
    Values = new[,] { { name, newModel.FindAnnotation("ProductVersion")?.Value ?? "Unknown version" } }
    };
    operations = operations.Append(insert).ToList();
    operations.Add(
    new SqlOperation(){
    Sql = insert
    });

    // Convert the operations to sql, then execute the operations
    var sqlGenerator = db.GetService<IMigrationsSqlGenerator>();
    @@ -185,43 +205,49 @@ private async Task<string> AutoMigrate(Assembly migrationAssembly, IModel oldMod
    }

    // Migrate the database by first applying release migrations, then by auto migrating from the model snapshot stored in the database
    // Returns true if any migrations were run
    public async Task<bool> Migrate()
    public async Task<MigrationResult> Migrate()
    {
    var ret = false;
    var ret = MigrationResult.Noop;

    var migrationAssembly = db.GetService<IMigrationsAssembly>();
    var migrations = db.Database.GetMigrations();
    var appliedMigrations = (await db.Database.GetAppliedMigrationsAsync()).ToList();
    var migrateDatabase = MigrateNewDatabase || migrations.Intersect(appliedMigrations).Any();

    var pendingMigrations = migrations.Except(appliedMigrations).Any();
    var pendingMigrations = migrateDatabase && migrations.Except(appliedMigrations).Any();
    var devMigration = appliedMigrations.Except(migrations).LastOrDefault();

    ModelSnapshot modelSnapshot;
    ModelSnapshot modelSnapshot = null;
    if (devMigration != null)
    {
    if (pendingMigrations)
    throw new InvalidOperationException("An automatic migration has been run, but you've added new release migration(s).\nYou'll need to restore from a release database.");

    var source = await ReadSnapshotSource();
    if (source == null || !source.Contains(devMigration))
    throw new InvalidOperationException($"Expected to find the source code of the {devMigration} ModelSnapshot stored in the database");

    modelSnapshot = CompileSnapshot(migrationAssembly.Assembly, source);
    }
    else
    {
    if (pendingMigrations)
    if (migrateDatabase)
    {
    // Run release migrations
    await db.Database.MigrateAsync();
    ret = true;
    if (pendingMigrations)
    {
    // Run release migrations
    await db.Database.MigrateAsync();
    ret = MigrationResult.Migrated;
    }

    modelSnapshot = migrationAssembly.ModelSnapshot;
    }

    modelSnapshot = migrationAssembly.ModelSnapshot;
    }

    var newSnapshot = await AutoMigrate(migrationAssembly.Assembly, modelSnapshot?.Model, db.Model);
    if (newSnapshot != null)
    {
    ret = true;
    ret = appliedMigrations.Any() ? MigrationResult.AutoMigrated : MigrationResult.Created;
    await WriteSnapshotSource(newSnapshot);
    }

  2. @lakeman lakeman created this gist Oct 23, 2019.
    232 changes: 232 additions & 0 deletions AutoMigration.cs
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,232 @@
    using Microsoft.CodeAnalysis;
    using Microsoft.CodeAnalysis.CSharp;
    using Microsoft.EntityFrameworkCore;
    using Microsoft.EntityFrameworkCore.Design.Internal;
    using Microsoft.EntityFrameworkCore.Infrastructure;
    using Microsoft.EntityFrameworkCore.Metadata;
    using Microsoft.EntityFrameworkCore.Migrations;
    using Microsoft.EntityFrameworkCore.Migrations.Design;
    using Microsoft.EntityFrameworkCore.Migrations.Operations;
    using Microsoft.EntityFrameworkCore.Storage;
    using Microsoft.Extensions.DependencyInjection;
    using Microsoft.Extensions.Logging;
    using System;
    using System.Collections.Generic;
    using System.Data.Common;
    using System.IO;
    using System.IO.Compression;
    using System.Linq;
    using System.Reflection;
    using System.Runtime;
    using System.Runtime.Loader;
    using System.Text;
    using System.Threading.Tasks;

    namespace ...
    {
    public class AutoMigration : IOperationReporter
    {
    private readonly DbContext db;
    private readonly ILogger logger;

    public AutoMigration(DbContext db, ILogger logger)
    {
    this.db = db;
    this.logger = logger;
    }

    public bool AllowDestructive { get; set; } = false;

    void IOperationReporter.WriteError(string message) => logger.LogError(message);
    void IOperationReporter.WriteInformation(string message) => logger.LogInformation(message);
    void IOperationReporter.WriteVerbose(string message) => logger.LogTrace(message);
    void IOperationReporter.WriteWarning(string message) => logger.LogWarning(message);

    private DbCommand newCmd()
    {
    var conn = db.Database.GetDbConnection();
    using var cmd = conn.CreateCommand();
    cmd.Transaction = db.Database.CurrentTransaction?.GetDbTransaction();
    return cmd;
    }

    // load the last model snapshot from the database
    private async Task<string> ReadSnapshotSource()
    {
    using var cmd = newCmd();
    cmd.CommandText = "select snapshot from auto_migration";

    await db.Database.OpenConnectionAsync();
    try
    {
    using var reader = cmd.ExecuteReader();

    if (!await reader.ReadAsync())
    return null;

    using var stream = new GZipStream(reader.GetStream(0), CompressionMode.Decompress);
    return await new StreamReader(stream).ReadToEndAsync();
    }
    finally
    {
    await db.Database.CloseConnectionAsync();
    }
    }

    private async Task WriteSnapshotSource(string source)
    {
    // write snapshot into the database
    await db.Database.ExecuteSqlRawAsync(
    @"IF NOT EXISTS (SELECT 1 FROM sysobjects WHERE name='auto_migration' and xtype='U')
    CREATE TABLE auto_migration (snapshot varbinary(max) null)"
    );
    await db.Database.ExecuteSqlRawAsync(
    @"insert into auto_migration(snapshot) select null where not exists(select 1 from auto_migration)"
    );

    using var dbStream = new MemoryStream();
    using (var blobStream = new GZipStream(dbStream, CompressionLevel.Fastest, true))
    {
    await blobStream.WriteAsync(Encoding.UTF8.GetBytes(source));
    }
    dbStream.Seek(0, SeekOrigin.Begin);

    await db.Database.ExecuteSqlInterpolatedAsync($"update auto_migration set snapshot = {dbStream.ToArray()}");
    }

    private T Compile<T>(string source, IEnumerable<Assembly> references)
    {
    var options = CSharpParseOptions.Default
    .WithLanguageVersion(LanguageVersion.Latest);

    var compileOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)
    .WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default);

    var compilation = CSharpCompilation.Create("Dynamic",
    new[] { SyntaxFactory.ParseSyntaxTree(source, options) },
    references.Select(a => MetadataReference.CreateFromFile(a.Location)),
    compileOptions
    );

    using var ms = new MemoryStream();
    var e = compilation.Emit(ms);
    if (!e.Success)
    throw new Exception("Compilation failed");
    ms.Seek(0, SeekOrigin.Begin);

    var context = new AssemblyLoadContext(null, true);
    var assembly = context.LoadFromStream(ms);

    var modelType = assembly.DefinedTypes.Where(t => typeof(T).IsAssignableFrom(t)).Single();

    return (T)Activator.CreateInstance(modelType);
    }

    private ModelSnapshot CompileSnapshot(Assembly migrationAssembly, string source) =>
    Compile<ModelSnapshot>(source, new HashSet<Assembly>() {
    AppDomain.CurrentDomain.GetAssemblies().Where(a => a.GetName().Name == "netstandard").Single(),
    typeof(object).Assembly,
    typeof(DbContext).Assembly,
    migrationAssembly,
    db.GetType().Assembly,
    typeof(DbContextAttribute).Assembly,
    typeof(ModelSnapshot).Assembly,
    typeof(SqlServerValueGenerationStrategy).Assembly,
    typeof(AssemblyTargetedPatchBandAttribute).Assembly
    });

    [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "EF1001:Internal EF Core API usage.", Justification = "Just because")]
    private async Task<string> AutoMigrate(Assembly migrationAssembly, IModel oldModel, IModel newModel)
    {
    var builder = new DesignTimeServicesBuilder(migrationAssembly, Assembly.GetEntryAssembly(), this, null);
    var services = builder.Build(db);
    var dependencies = services.GetRequiredService<MigrationsScaffolderDependencies>();
    var name = $"{DateTime.Now:yyyyMMddHHmmss}_Auto";

    if (oldModel == null)
    {
    //(not tested...)
    await db.Database.EnsureCreatedAsync();
    }
    else
    {
    // apply fixes for upgrading between major / minor versions
    oldModel = dependencies.SnapshotModelProcessor.Process(oldModel);

    var operations = dependencies.MigrationsModelDiffer.GetDifferences(oldModel, newModel);

    if (!operations.Any())
    return null;

    if (!AllowDestructive && operations.Any(o => o.IsDestructiveChange))
    throw new InvalidOperationException(
    "Automatic migration was not applied because it could result in data loss.");

    // insert an extra step to track the history of auto migrations
    var insert = new InsertDataOperation()
    {
    Table = "__EFMigrationsHistory", // TODO the table name can be overridden in config
    Schema = "dbo",
    Columns = new[] { nameof(HistoryRow.MigrationId), nameof(HistoryRow.ProductVersion) },
    Values = new[,] { { name, newModel.FindAnnotation("ProductVersion")?.Value ?? "Unknown version" } }
    };
    operations = operations.Append(insert).ToList();

    // Convert the operations to sql, then execute the operations
    var sqlGenerator = db.GetService<IMigrationsSqlGenerator>();
    var commands = sqlGenerator.Generate(operations, db.Model);
    var executor = db.GetService<IMigrationCommandExecutor>();

    await executor.ExecuteNonQueryAsync(commands, db.GetService<IRelationalConnection>());
    }

    var codeGen = dependencies.MigrationsCodeGeneratorSelector.Select(null);
    return codeGen.GenerateSnapshot("AutoMigrations", db.GetType(), $"Migration_{name}", newModel);
    }

    // Migrate the database by first applying release migrations, then by auto migrating from the model snapshot stored in the database
    // Returns true if any migrations were run
    public async Task<bool> Migrate()
    {
    var ret = false;

    var migrationAssembly = db.GetService<IMigrationsAssembly>();
    var migrations = db.Database.GetMigrations();
    var appliedMigrations = (await db.Database.GetAppliedMigrationsAsync()).ToList();

    var pendingMigrations = migrations.Except(appliedMigrations).Any();
    var devMigration = appliedMigrations.Except(migrations).LastOrDefault();

    ModelSnapshot modelSnapshot;
    if (devMigration != null)
    {
    if (pendingMigrations)
    throw new InvalidOperationException("An automatic migration has been run, but you've added new release migration(s).\nYou'll need to restore from a release database.");

    var source = await ReadSnapshotSource();
    modelSnapshot = CompileSnapshot(migrationAssembly.Assembly, source);
    }
    else
    {
    if (pendingMigrations)
    {
    // Run release migrations
    await db.Database.MigrateAsync();
    ret = true;
    }

    modelSnapshot = migrationAssembly.ModelSnapshot;
    }

    var newSnapshot = await AutoMigrate(migrationAssembly.Assembly, modelSnapshot?.Model, db.Model);
    if (newSnapshot != null)
    {
    ret = true;
    await WriteSnapshotSource(newSnapshot);
    }

    return ret;
    }
    }
    }