Model save

README.md

@@ -0,0 +1,71 @@
+---
+license: mit
+base_model: microsoft/deberta-v3-base
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+- f1
+- precision
+- recall
+model-index:
+- name: deberta-v3-base-orgs-v2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deberta-v3-base-orgs-v2
+
+This model is a fine-tuned version of [microsoft/deberta-v3-base](https://huggingface.co/microsoft/deberta-v3-base) on the None dataset.
+It achieves the following results on the evaluation set:
+- Accuracy: 0.9632
+- F1: 0.7927
+- Loss: 0.1186
+- Precision: 0.8127
+- Recall: 0.7735
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 256
+- eval_batch_size: 256
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 20
+- num_epochs: 3.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Accuracy | F1     | Validation Loss | Precision | Recall |
+|:-------------:|:-----:|:----:|:--------:|:------:|:---------------:|:---------:|:------:|
+| 0.0706        | 0.7   | 600  | 0.9602   | 0.7690 | 0.1138          | 0.7590    | 0.7793 |
+| 0.0526        | 1.4   | 1200 | 0.9617   | 0.7870 | 0.1113          | 0.7942    | 0.7799 |
+| 0.0409        | 2.11  | 1800 | 0.9627   | 0.7875 | 0.1125          | 0.7911    | 0.7839 |
+| 0.0376        | 2.81  | 2400 | 0.9632   | 0.7927 | 0.1186          | 0.8127    | 0.7735 |
+
+
+### Framework versions
+
+- Transformers 4.35.2
+- Pytorch 2.1.0a0+32f93b1
+- Datasets 2.15.0
+- Tokenizers 0.15.0

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

all_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9627981870229008,
+    "eval_f1": 0.7875191815856777,
+    "eval_loss": 0.12025153636932373,
+    "eval_precision": 0.8050617025726835,
+    "eval_recall": 0.7707248698438126,
+    "eval_runtime": 3.4859,
+    "eval_samples": 3000,
+    "eval_samples_per_second": 860.608,
+    "eval_steps_per_second": 3.442,
+    "train_loss": 0.056466960860507065,
+    "train_runtime": 473.6102,
+    "train_samples": 109424,
+    "train_samples_per_second": 693.127,
+    "train_steps_per_second": 5.416
+}

config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-base",
+  "architectures": [
+    "DebertaV2ForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "ner",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-ORG",
+    "2": "I-ORG"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-ORG": 1,
+    "I-ORG": 2,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9627981870229008,
+    "eval_f1": 0.7875191815856777,
+    "eval_loss": 0.12025153636932373,
+    "eval_precision": 0.8050617025726835,
+    "eval_recall": 0.7707248698438126,
+    "eval_runtime": 3.4859,
+    "eval_samples": 3000,
+    "eval_samples_per_second": 860.608,
+    "eval_steps_per_second": 3.442
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e3414be3d3106da0080167cdaba17c9bef17ad636ab5438c5ce8d273428f07
+size 735359804

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.056466960860507065,
+    "train_runtime": 473.6102,
+    "train_samples": 109424,
+    "train_samples_per_second": 693.127,
+    "train_steps_per_second": 5.416
+}

trainer_state.json

@@ -0,0 +1,382 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 600,
+  "global_step": 2565,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.820662768031189e-05,
+      "loss": 0.2265,
+      "step": 50
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 9.625730994152047e-05,
+      "loss": 0.0839,
+      "step": 100
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 9.430799220272904e-05,
+      "loss": 0.0794,
+      "step": 150
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 9.235867446393762e-05,
+      "loss": 0.0751,
+      "step": 200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 9.04093567251462e-05,
+      "loss": 0.0733,
+      "step": 250
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 8.846003898635477e-05,
+      "loss": 0.0706,
+      "step": 300
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 8.651072124756335e-05,
+      "loss": 0.0695,
+      "step": 350
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 8.456140350877193e-05,
+      "loss": 0.071,
+      "step": 400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 8.26120857699805e-05,
+      "loss": 0.0691,
+      "step": 450
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.066276803118908e-05,
+      "loss": 0.0668,
+      "step": 500
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 7.871345029239767e-05,
+      "loss": 0.0659,
+      "step": 550
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 7.676413255360623e-05,
+      "loss": 0.0706,
+      "step": 600
+    },
+    {
+      "epoch": 0.7,
+      "eval_accuracy": 0.9601979961832061,
+      "eval_f1": 0.769017980636238,
+      "eval_loss": 0.11379604041576385,
+      "eval_precision": 0.7589703588143526,
+      "eval_recall": 0.7793352022426913,
+      "eval_runtime": 3.4889,
+      "eval_samples_per_second": 859.867,
+      "eval_steps_per_second": 3.439,
+      "step": 600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.481481481481481e-05,
+      "loss": 0.0689,
+      "step": 650
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 7.28654970760234e-05,
+      "loss": 0.0658,
+      "step": 700
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.091617933723196e-05,
+      "loss": 0.0645,
+      "step": 750
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.896686159844055e-05,
+      "loss": 0.0663,
+      "step": 800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.701754385964913e-05,
+      "loss": 0.063,
+      "step": 850
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 6.50682261208577e-05,
+      "loss": 0.0537,
+      "step": 900
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 6.311890838206628e-05,
+      "loss": 0.055,
+      "step": 950
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 6.116959064327486e-05,
+      "loss": 0.054,
+      "step": 1000
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 5.9220272904483434e-05,
+      "loss": 0.0532,
+      "step": 1050
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 5.727095516569201e-05,
+      "loss": 0.0534,
+      "step": 1100
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 5.5321637426900584e-05,
+      "loss": 0.0516,
+      "step": 1150
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 5.3372319688109166e-05,
+      "loss": 0.0526,
+      "step": 1200
+    },
+    {
+      "epoch": 1.4,
+      "eval_accuracy": 0.9617127862595419,
+      "eval_f1": 0.7870276823600727,
+      "eval_loss": 0.11129256337881088,
+      "eval_precision": 0.7942495921696574,
+      "eval_recall": 0.7799359231077293,
+      "eval_runtime": 3.4578,
+      "eval_samples_per_second": 867.603,
+      "eval_steps_per_second": 3.47,
+      "step": 1200
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 5.142300194931774e-05,
+      "loss": 0.0512,
+      "step": 1250
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 4.9473684210526315e-05,
+      "loss": 0.0515,
+      "step": 1300
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 4.752436647173489e-05,
+      "loss": 0.048,
+      "step": 1350
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 4.557504873294347e-05,
+      "loss": 0.0509,
+      "step": 1400
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 4.362573099415205e-05,
+      "loss": 0.0495,
+      "step": 1450
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 4.167641325536062e-05,
+      "loss": 0.0531,
+      "step": 1500
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 3.97270955165692e-05,
+      "loss": 0.0524,
+      "step": 1550
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 0.0496,
+      "step": 1600
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 3.582846003898635e-05,
+      "loss": 0.0487,
+      "step": 1650
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 3.3879142300194935e-05,
+      "loss": 0.0514,
+      "step": 1700
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 3.192982456140351e-05,
+      "loss": 0.0406,
+      "step": 1750
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 2.9980506822612088e-05,
+      "loss": 0.0409,
+      "step": 1800
+    },
+    {
+      "epoch": 2.11,
+      "eval_accuracy": 0.9626550572519084,
+      "eval_f1": 0.7874886855073921,
+      "eval_loss": 0.1125219315290451,
+      "eval_precision": 0.7910689028086482,
+      "eval_recall": 0.7839407288746496,
+      "eval_runtime": 3.4674,
+      "eval_samples_per_second": 865.198,
+      "eval_steps_per_second": 3.461,
+      "step": 1800
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 2.8031189083820663e-05,
+      "loss": 0.0386,
+      "step": 1850
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 2.608187134502924e-05,
+      "loss": 0.0387,
+      "step": 1900
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 2.413255360623782e-05,
+      "loss": 0.0399,
+      "step": 1950
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 2.2183235867446394e-05,
+      "loss": 0.0379,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 2.0233918128654973e-05,
+      "loss": 0.038,
+      "step": 2050
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 1.8284600389863547e-05,
+      "loss": 0.04,
+      "step": 2100
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 1.6335282651072126e-05,
+      "loss": 0.0382,
+      "step": 2150
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 1.4385964912280702e-05,
+      "loss": 0.0384,
+      "step": 2200
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 1.243664717348928e-05,
+      "loss": 0.038,
+      "step": 2250
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 1.0487329434697857e-05,
+      "loss": 0.0399,
+      "step": 2300
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 8.538011695906434e-06,
+      "loss": 0.0379,
+      "step": 2350
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 6.588693957115009e-06,
+      "loss": 0.0376,
+      "step": 2400
+    },
+    {
+      "epoch": 2.81,
+      "eval_accuracy": 0.9631560114503817,
+      "eval_f1": 0.7926541499948702,
+      "eval_loss": 0.11860690265893936,
+      "eval_precision": 0.8127498422049232,
+      "eval_recall": 0.7735282338806568,
+      "eval_runtime": 3.4445,
+      "eval_samples_per_second": 870.961,
+      "eval_steps_per_second": 3.484,
+      "step": 2400
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 4.639376218323587e-06,
+      "loss": 0.036,
+      "step": 2450
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 2.690058479532164e-06,
+      "loss": 0.0366,
+      "step": 2500
+    },
+    {
+      "epoch": 2.98,
+      "learning_rate": 7.407407407407408e-07,
+      "loss": 0.0375,
+      "step": 2550
+    },
+    {
+      "epoch": 3.0,
+      "step": 2565,
+      "total_flos": 1.7165637031885056e+16,
+      "train_loss": 0.056466960860507065,
+      "train_runtime": 473.6102,
+      "train_samples_per_second": 693.127,
+      "train_steps_per_second": 5.416
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 2565,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 1.7165637031885056e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8477dc023ec1022bcc64d9971312e6e9e0f93be242ffa235784eb729a3626e4
+size 4600