001package io.ebean.migration.runner;
002
003import io.ebean.migration.JdbcMigration;
004import io.ebean.migration.MigrationConfig;
005import io.ebean.migration.MigrationException;
006import io.ebean.migration.MigrationVersion;
007import io.ebean.migration.util.IOUtils;
008import org.slf4j.Logger;
009import org.slf4j.LoggerFactory;
010
011import java.io.IOException;
012import java.net.URL;
013import java.sql.Connection;
014import java.sql.DatabaseMetaData;
015import java.sql.ResultSet;
016import java.sql.SQLException;
017import java.sql.Timestamp;
018import java.util.ArrayList;
019import java.util.Enumeration;
020import java.util.LinkedHashMap;
021import java.util.List;
022import java.util.Set;
023
024import static io.ebean.migration.MigrationVersion.BOOTINIT_TYPE;
025
026/**
027 * Manages the migration table.
028 */
029public class MigrationTable {
030
031  private static final Logger logger = LoggerFactory.getLogger("io.ebean.DDL");
032
033  private static final String INIT_VER_0 = "0";
034
035  private final Connection connection;
036  private final boolean checkState;
037
038  private final String catalog;
039  private final String schema;
040  private final String table;
041  private final String sqlTable;
042  private final String envUserName;
043  private final String platformName;
044
045  private final Timestamp runOn = new Timestamp(System.currentTimeMillis());
046
047  private final ScriptTransform scriptTransform;
048
049  private final String insertSql;
050  private final String updateSql;
051  private final String updateChecksumSql;
052
053  private final LinkedHashMap<String, MigrationMetaRow> migrations;
054  private final boolean skipChecksum;
055
056  private final Set<String> patchInsertVersions;
057  private final Set<String> patchResetChecksumVersions;
058  private final boolean allowErrorInRepeatable;
059
060  private MigrationMetaRow lastMigration;
061  private LocalMigrationResource priorVersion;
062
063  private final List<LocalMigrationResource> checkMigrations = new ArrayList<>();
064
065  /**
066   * Version of a dbinit script. When set this means all migration version less than this are ignored.
067   */
068  private MigrationVersion dbInitVersion;
069
070  /**
071   * Construct with server, configuration and jdbc connection (DB admin user).
072   */
073  public MigrationTable(MigrationConfig config, Connection connection, boolean checkState) {
074
075    this.connection = connection;
076    this.checkState = checkState;
077    this.migrations = new LinkedHashMap<>();
078
079    this.catalog = null;
080    this.allowErrorInRepeatable = config.isAllowErrorInRepeatable();
081    this.patchResetChecksumVersions = config.getPatchResetChecksumOn();
082    this.patchInsertVersions = config.getPatchInsertOn();
083    this.skipChecksum = config.isSkipChecksum();
084    this.schema = config.getDbSchema();
085    this.table = config.getMetaTable();
086    this.platformName = config.getPlatformName();
087    this.sqlTable = initSqlTable();
088    this.insertSql = MigrationMetaRow.insertSql(sqlTable);
089    this.updateSql = MigrationMetaRow.updateSql(sqlTable);
090    this.updateChecksumSql = MigrationMetaRow.updateChecksumSql(sqlTable);
091    this.scriptTransform = createScriptTransform(config);
092    this.envUserName = System.getProperty("user.name");
093  }
094
095  private String initSqlTable() {
096    if (schema != null) {
097      return schema + "." + table;
098    } else {
099      return table;
100    }
101  }
102
103  private String sqlPrimaryKey() {
104    return "pk_" + table;
105  }
106
107  /**
108   * Return the number of migrations in the DB migration table.
109   */
110  public int size() {
111    return migrations.size();
112  }
113
114  /**
115   * Returns the versions that are already applied.
116   */
117  public Set<String> getVersions() {
118    return migrations.keySet();
119  }
120
121  /**
122   * Create the ScriptTransform for placeholder key/value replacement.
123   */
124  private ScriptTransform createScriptTransform(MigrationConfig config) {
125
126    return ScriptTransform.build(config.getRunPlaceholders(), config.getRunPlaceholderMap());
127  }
128
129  /**
130   * Create the table is it does not exist.
131   * <p>
132   * Also holds DB lock on migration table and loads existing migrations.
133   * </p>
134   */
135  public void createIfNeededAndLock(MigrationPlatform platform) throws SQLException, IOException {
136
137    if (!tableExists(connection)) {
138      createTable(connection);
139    }
140    obtainLockWithWait(platform);
141    readExistingMigrations(platform);
142  }
143
144  /**
145   * Obtain lock with wait, note that other nodes can insert and commit
146   * into the migration table during the wait so this query result won't
147   * contain all the executed migrations in that case.
148   */
149  private void obtainLockWithWait(MigrationPlatform platform) throws SQLException {
150    platform.lockMigrationTable(sqlTable, connection);
151  }
152
153  /**
154   * Read the migration table with details on what migrations have run.
155   * This must execute after we have completed the wait for the lock on
156   * the migration table such that it reads any migrations that have
157   * executed during the wait for the lock.
158   */
159  private void readExistingMigrations(MigrationPlatform platform) throws SQLException {
160    for (MigrationMetaRow metaRow : platform.readExistingMigrations(sqlTable, connection)) {
161      addMigration(metaRow.getVersion(), metaRow);
162    }
163  }
164
165  private void createTable(Connection connection) throws IOException, SQLException {
166
167    MigrationScriptRunner run = new MigrationScriptRunner(connection);
168    run.runScript(false, createTableDdl(), "create migration table");
169    createInitMetaRow().executeInsert(connection, insertSql);
170  }
171
172  /**
173   * Return the create table script.
174   */
175  String createTableDdl() throws IOException {
176    String script = ScriptTransform.replace("${table}", sqlTable, getCreateTableScript());
177    return ScriptTransform.replace("${pk_table}", sqlPrimaryKey(), script);
178  }
179
180  /**
181   * Return the create table script.
182   */
183  private String getCreateTableScript() throws IOException {
184    // supply a script to override the default table create script
185    String script = readResource("migration-support/create-table.sql");
186    if (script == null && platformName != null && !platformName.isEmpty()) {
187      // look for platform specific create table
188      script = readResource("migration-support/" + platformName + "-create-table.sql");
189    }
190    if (script == null) {
191      // no, just use the default script
192      script = readResource("migration-support/default-create-table.sql");
193    }
194    return script;
195  }
196
197  private String readResource(String location) throws IOException {
198
199    Enumeration<URL> resources = getClassLoader().getResources(location);
200    if (resources.hasMoreElements()) {
201      URL url = resources.nextElement();
202      return IOUtils.readUtf8(url);
203    }
204    return null;
205  }
206
207  private ClassLoader getClassLoader() {
208    return Thread.currentThread().getContextClassLoader();
209  }
210
211  /**
212   * Return true if the table exists.
213   */
214  private boolean tableExists(Connection connection) throws SQLException {
215
216    String migTable = table;
217
218    DatabaseMetaData metaData = connection.getMetaData();
219    if (metaData.storesUpperCaseIdentifiers()) {
220      migTable = migTable.toUpperCase();
221    }
222    String checkCatalog = (catalog != null) ? catalog : connection.getCatalog();
223    String checkSchema = (schema != null) ? schema : connection.getSchema();
224    try (ResultSet tables = metaData.getTables(checkCatalog, checkSchema, migTable, null)) {
225      return tables.next();
226    }
227  }
228
229  /**
230   * Return true if the migration ran successfully and false if the migration failed.
231   */
232  private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource prior) throws SQLException {
233
234    if (prior != null && !localVersion.isRepeatable()) {
235      if (!migrationExists(prior)) {
236        logger.error("Migration {} requires prior migration {} which has not been run", localVersion.getVersion(), prior.getVersion());
237        return false;
238      }
239    }
240
241    MigrationMetaRow existing = migrations.get(localVersion.key());
242    if (!runMigration(localVersion, existing)) {
243      return false;
244    }
245
246    // migration was run successfully ...
247    priorVersion = localVersion;
248    return true;
249  }
250
251  /**
252   * Run the migration script.
253   *
254   * @param local    The local migration resource
255   * @param existing The information for this migration existing in the table
256   * @return True if the migrations should continue
257   */
258  private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
259
260    String script = null;
261    int checksum;
262    if (local instanceof LocalDdlMigrationResource) {
263      script = convertScript(local.getContent());
264      checksum = Checksum.calculate(script);
265    } else {
266      checksum = ((LocalJdbcMigrationResource) local).getChecksum();
267    }
268
269    if (existing == null && patchInsertMigration(local, checksum)) {
270      return true;
271    }
272    if (existing != null && skipMigration(checksum, local, existing)) {
273      return true;
274    }
275    executeMigration(local, script, checksum, existing);
276    return true;
277  }
278
279  /**
280   * Return true if we 'patch history' inserting a DB migration without running it.
281   */
282  private boolean patchInsertMigration(LocalMigrationResource local, int checksum) throws SQLException {
283    if (patchInsertVersions != null && patchInsertVersions.contains(local.key())) {
284      logger.info("patch migration - insert into history {}", local.getLocation());
285      if (!checkState) {
286        insertIntoHistory(local, checksum, 0);
287      }
288      return true;
289    }
290    return false;
291  }
292
293  /**
294   * Return true if the migration should be skipped.
295   */
296  boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
297
298    boolean matchChecksum = (existing.getChecksum() == checksum);
299    if (matchChecksum) {
300      logger.trace("... skip unchanged migration {}", local.getLocation());
301      return true;
302
303    } else if (patchResetChecksum(existing, checksum)) {
304      logger.info("patch migration - reset checksum on {}", local.getLocation());
305      return true;
306
307    } else if (local.isRepeatable() || skipChecksum) {
308      // re-run the migration
309      return false;
310    } else {
311      throw new MigrationException("Checksum mismatch on migration " + local.getLocation());
312    }
313  }
314
315  /**
316   * Return true if the checksum is reset on the existing migration.
317   */
318  private boolean patchResetChecksum(MigrationMetaRow existing, int newChecksum) throws SQLException {
319
320    if (isResetOnVersion(existing.getVersion())) {
321      if (!checkState) {
322        existing.resetChecksum(newChecksum, connection, updateChecksumSql);
323      }
324      return true;
325    } else {
326      return false;
327    }
328  }
329
330  private boolean isResetOnVersion(String version) {
331    return patchResetChecksumVersions != null && patchResetChecksumVersions.contains(version);
332  }
333
334  /**
335   * Run a migration script as new migration or update on existing repeatable migration.
336   */
337  private void executeMigration(LocalMigrationResource local, String script, int checksum, MigrationMetaRow existing) throws SQLException {
338
339    if (checkState) {
340      checkMigrations.add(local);
341      // simulate the migration being run such that following migrations also match
342      addMigration(local.key(), createMetaRow(local, checksum, 1));
343      return;
344    }
345
346    logger.debug("run migration {}", local.getLocation());
347
348    long start = System.currentTimeMillis();
349    try {
350      if (local instanceof LocalDdlMigrationResource) {
351        MigrationScriptRunner run = new MigrationScriptRunner(connection);
352        run.runScript(false, script, "run migration version: " + local.getVersion());
353      } else {
354        JdbcMigration migration = ((LocalJdbcMigrationResource) local).getMigration();
355        logger.info("Executing jdbc migration version: {} - {}", local.getVersion(), migration);
356        migration.migrate(connection);
357      }
358      long exeMillis = System.currentTimeMillis() - start;
359
360      if (existing != null) {
361        existing.rerun(checksum, exeMillis, envUserName, runOn);
362        existing.executeUpdate(connection, updateSql);
363
364      } else {
365        insertIntoHistory(local, checksum, exeMillis);
366      }
367    } catch (SQLException e) {
368      if (allowErrorInRepeatable && local.isRepeatableLast()) {
369        // log the exception and continue on repeatable migration
370        logger.error("Continue migration with error executing repeatable migration " + local.getVersion(), e);
371      } else {
372        throw e;
373      }
374    }
375  }
376
377  private void insertIntoHistory(LocalMigrationResource local, int checksum, long exeMillis) throws SQLException {
378    MigrationMetaRow metaRow = createMetaRow(local, checksum, exeMillis);
379    metaRow.executeInsert(connection, insertSql);
380    addMigration(local.key(), metaRow);
381  }
382
383  private MigrationMetaRow createInitMetaRow() {
384    return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0);
385  }
386
387  /**
388   * Create the MigrationMetaRow for this migration.
389   */
390  private MigrationMetaRow createMetaRow(LocalMigrationResource migration, int checksum, long exeMillis) {
391
392    int nextId = 1;
393    if (lastMigration != null) {
394      nextId = lastMigration.getId() + 1;
395    }
396
397    String type = migration.getType();
398    String runVersion = migration.key();
399    String comment = migration.getComment();
400
401    return new MigrationMetaRow(nextId, type, runVersion, comment, checksum, envUserName, runOn, exeMillis);
402  }
403
404  /**
405   * Return true if the migration exists.
406   */
407  private boolean migrationExists(LocalMigrationResource priorVersion) {
408    return migrations.containsKey(priorVersion.key());
409  }
410
411  /**
412   * Apply the placeholder key/value replacement on the script.
413   */
414  private String convertScript(String script) {
415    return scriptTransform.transform(script);
416  }
417
418  /**
419   * Register the successfully executed migration (to allow dependant scripts to run).
420   */
421  private void addMigration(String key, MigrationMetaRow metaRow) {
422    if (INIT_VER_0.equals(key)) {
423      // ignore the version 0 <init> row
424      return;
425    }
426    lastMigration = metaRow;
427    if (metaRow.getVersion() == null) {
428      throw new IllegalStateException("No runVersion in db migration table row? " + metaRow);
429    }
430    migrations.put(key, metaRow);
431    if (BOOTINIT_TYPE.equals(metaRow.getType())) {
432      dbInitVersion = MigrationVersion.parse(metaRow.getVersion());
433    }
434  }
435
436  /**
437   * Return true if there are no migrations.
438   */
439  public boolean isEmpty() {
440    return migrations.isEmpty();
441  }
442
443  /**
444   * Run all the migrations in order as needed.
445   *
446   * @return the migrations that have been run (collected if checkstate is true).
447   */
448  public List<LocalMigrationResource> runAll(List<LocalMigrationResource> localVersions) throws SQLException {
449    for (LocalMigrationResource localVersion : localVersions) {
450      if (!localVersion.isRepeatable() && dbInitVersion != null && dbInitVersion.compareTo(localVersion.getVersion()) >= 0) {
451        logger.debug("migration skipped by dbInitVersion {}", dbInitVersion);
452      } else if (!shouldRun(localVersion, priorVersion)) {
453        break;
454      }
455    }
456    return checkMigrations;
457  }
458
459  /**
460   * Run using an init migration.
461   *
462   * @return the migrations that have been run (collected if checkstate is true).
463   */
464  public List<LocalMigrationResource> runInit(LocalMigrationResource initVersion, List<LocalMigrationResource> localVersions) throws SQLException {
465
466    runRepeatableInit(localVersions);
467
468    initVersion.setInitType();
469    if (!shouldRun(initVersion, null)) {
470      throw new IllegalStateException("Expected to run init migration but it didn't?");
471    }
472
473    // run any migrations greater that the init migration
474    for (LocalMigrationResource localVersion : localVersions) {
475      if (localVersion.compareTo(initVersion) > 0 && !shouldRun(localVersion, priorVersion)) {
476        break;
477      }
478    }
479    return checkMigrations;
480  }
481
482  private void runRepeatableInit(List<LocalMigrationResource> localVersions) throws SQLException {
483    for (LocalMigrationResource localVersion : localVersions) {
484      if (!localVersion.isRepeatableInit() || !shouldRun(localVersion, priorVersion)) {
485        break;
486      }
487    }
488  }
489
490}