JPQ_ActivityNet.json

{
    "n_gpu": 1,
    "seed": 0,
    "experts": {
        "face_dim": 512,
        "modalities": [
            "s3d",
            "vggish"
        ]
    },
    "arch": {
        "type": "JPQ",
        "args": {
            "keep_missing_modalities": true,
            "test_caption_mode": "indep",
            "txt_inp": "bertftn",
            "txt_agg": "bertftn",
            "txt_wgh": "emb",
            "vid_inp": "both",
            "pos_enc": "tint",
            "out_tok": "mxp",
            "l2renorm": false,
            "vid_bert_params": {
                "vocab_size_or_config_json_file": 10,
                "hidden_size": 512,
                "num_hidden_layers": 4,
                "num_attention_heads": 4,
                "intermediate_size": 3072,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "attention_probs_dropout_prob": 0.1,
                "max_position_embeddings": 102,
                "type_vocab_size": 19,
                "initializer_range": 0.02,
                "layer_norm_eps": 1e-12
            },
            "txt_pro": "gbn",
            "txt_bert_params": {
                "hidden_dropout_prob": 0.2,
                "attention_probs_dropout_prob": 0.2
            },
            "pq_params": {
                "feat_dim": 512,
                "M": 256,
                "K": 256,
                "mask_topK": null,
                "alpha": 1,
                "update_codebooks": true,
                "dual_codebooks": false,
                "add_BN": true,
                "pq_detach_opt": "none"
            }
        }
    },
    "train_sets": [
        {
            "type": "ExpertDataLoader",
            "args": {
                "mix": [
                    {
                        "dataset_name": "ActivityNet",
                        "cut_name": "val1",
                        "split_name": "trainval",
                        "data_dir": "./data/activity-net/symlinked-feats",
                        "mix_weight": 1.0,
                        "query_shuffling": "cat",
                        "temporal_encoding_window": 1,
                        "max_text_words": 100,
                        "max_expert_tokens": 100
                    }
                ],
                "batch_size": 32,
                "num_workers": 64,
                "pin_memory": false
            }
        }
    ],
    "continuous_eval_sets": [
        {
            "type": "ExpertDataLoader",
            "args": {
                "mix": [
                    {
                        "dataset_name": "ActivityNet",
                        "cut_name": "val1",
                        "split_name": "test",
                        "data_dir": "./data/activity-net/symlinked-feats",
                        "captions_per_video": 1,
                        "query_shuffling": "cat",
                        "temporal_encoding_window": 1,
                        "max_text_words": 100,
                        "max_expert_tokens": 100
                    }
                ],
                "batch_size": 32,
                "num_workers": 64,
                "pin_memory": false
            }
        }
    ],
    "final_eval_sets": [
        {
            "type": "ExpertDataLoader",
            "args": {
                "mix": [
                    {
                        "dataset_name": "ActivityNet",
                        "cut_name": "val1",
                        "split_name": "test",
                        "data_dir": "./data/activity-net/symlinked-feats",
                        "captions_per_video": 1,
                        "query_shuffling": "cat",
                        "temporal_encoding_window": 1,
                        "max_text_words": 100,
                        "max_expert_tokens": 100
                    }
                ],
                "batch_size": 32,
                "num_workers": 0,
                "pin_memory": false
            }
        }
    ],
    "optimizer": {
        "type": "Adam",
        "args": {
            "lr": 5e-05,
            "weight_decay": 0
        }
    },
    "loss": {
        "type": "QuantizedRankingLoss",
        "args": {
            "margin": 0.07,
            "total_epochs": 50,
            "pq_reg_loss_weight": 0.0
        }
    },
    "metrics": [
        "t2v_metrics",
        "v2t_metrics"
    ],
    "visualizer": {
        "type": "Visualizer",
        "args": {
            "vis_vid_freq": 25.0,
            "num_samples": 20
        }
    },
    "lr_scheduler": {
        "type": "StepLR",
        "args": {
            "step_size": 1,
            "gamma": 0.9
        }
    },
    "trainer": {
        "epochs": 50,
        "max_samples_per_epoch": 32000,
        "save_period": 1,
        "skip_first_n_saves": 0,
        "codebook_fixed_epoches": 0,
        "update_codebooks_by_clustering": false,
        "include_optim_in_ckpts": true,
        "verbosity": 2,
        "tensorboard": true,
        "monitor": "max ActivityNet_val1_test/t2v_metrics/geometric_mean_R1-R5-R10"
    }
}