utils.py

import tensorflow as tf
import numpy as np
import scipy.ndimage
import logging

def set_mixed_precision_policy(strategy: tf.distribute.Strategy, use_mixed_precision: bool = True):
    if use_mixed_precision:
        if isinstance(strategy, tf.distribute.TPUStrategy):
            tf.keras.mixed_precision.set_global_policy('mixed_bfloat16')
        else:
            # TODO; tf.keras.mixed_precision.LossScaleOptimizer
            tf.keras.mixed_precision.set_global_policy('mixed_float16')
    else:
        tf.keras.mixed_precision.set_global_policy('float32')


def set_pretrained_pos_embed_for_vit(backbone, ckpt_path):
    reader = tf.train.load_checkpoint(ckpt_path)
    var_shape_map = reader.get_variable_to_shape_map()
    key = [key for key in var_shape_map if key.startswith('backbone/pos_emb') and not 'optimizer' in key]
    assert len(key) == 1, "cannot find positional embedding layer ('pos_emb')"
    posemb = reader.get_tensor(key[0])
    posemb_new = backbone.pos_emb.numpy()
    logging.info(f"load pretrained: resized variant: {posemb.shape} to {posemb_new.shape}")

    if posemb.shape[1] != posemb_new.shape[1]:
        ntok_new = posemb_new.shape[1] - 1
        posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:]

        gs_old = int(np.sqrt(len(posemb_grid)))
        gs_new = int(np.sqrt(ntok_new))
        logging.info(f"load pretrained: grid-size from {gs_old} to {gs_new}")
        posemb_grid = posemb_grid.reshape(gs_old, gs_old, -1)

        zoom = (gs_new / gs_old, gs_new / gs_old, 1)
        posemb_grid = scipy.ndimage.zoom(posemb_grid, zoom, order=1)
        posemb_grid = posemb_grid.reshape(1, gs_new * gs_new, -1)
        embedding_weights = tf.convert_to_tensor(
            np.concatenate([posemb_tok, posemb_grid], axis=1)
        )
    else:
        embedding_weights = posemb
    backbone.pos_emb.assign(embedding_weights)