Training in progress, step 712
Browse files- logs/amlt_code_runner.txt +20 -20
- model-00001-of-00003.safetensors +1 -1
- model-00002-of-00003.safetensors +1 -1
- model-00003-of-00003.safetensors +1 -1
- training_args.bin +1 -1
logs/amlt_code_runner.txt
CHANGED
|
@@ -1,21 +1,21 @@
|
|
| 1 |
-
2025-04-10 10:
|
| 2 |
-
2025-04-10 10:
|
| 3 |
-
2025-04-10 10:
|
| 4 |
-
2025-04-10 10:
|
| 5 |
-
2025-04-10 10:
|
| 6 |
-
2025-04-10 10:
|
| 7 |
-
2025-04-10 10:
|
| 8 |
-
2025-04-10 10:
|
| 9 |
-
2025-04-10 10:
|
| 10 |
-
2025-04-10 10:
|
| 11 |
-
2025-04-10 10:
|
| 12 |
-
2025-04-10 10:
|
| 13 |
-
2025-04-10 10:
|
| 14 |
-
2025-04-10 10:
|
| 15 |
-
2025-04-10 10:
|
| 16 |
|
| 17 |
-
2025-04-10 10:
|
| 18 |
-
2025-04-10 10:
|
| 19 |
-
2025-04-10 10:
|
| 20 |
-
2025-04-10 10:
|
| 21 |
-
2025-04-10 10:
|
|
|
|
| 1 |
+
2025-04-10 10:30:01,231:amlt-code-runner:INFO - SINGULARITY_LOCATION: centralus
|
| 2 |
+
2025-04-10 10:30:01,231:amlt-code-runner:INFO - AISC_INSTANCE_TYPE: Singularity.ND96_v4
|
| 3 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - Not removing AzureML's cd commands from /etc/profile due to an error: [Errno 13] Permission denied: '/etc/profile'
|
| 4 |
+
2025-04-10 10:30:04,308:amlt-code-runner:WARNING - Environment variable 'NCCL_SOCKET_IFNAME' already set to '=eth0', not changing to '^docker0,lo'
|
| 5 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - RANK = 0
|
| 6 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - LOCAL_RANK = None
|
| 7 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - WORLD_SIZE = 1
|
| 8 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - MASTER_ADDR = node-0
|
| 9 |
+
2025-04-10 10:30:04,308:amlt-code-runner:INFO - MASTER_PORT = 9500
|
| 10 |
+
2025-04-10 10:30:04,310:amlt-code-runner:WARNING - Installing amlt runtime dependencies: ['wrapt', 'azure-identity', 'python-dateutil', 'pytz'] into /tmp/amlt-user-base
|
| 11 |
+
2025-04-10 10:30:05,864:amlt-code-runner:INFO - Setting WANDB_RUN_ID to 'sincere_feast_96p6v4qzfp_156'
|
| 12 |
+
2025-04-10 10:30:05,864:amlt-code-runner:INFO - Expanding HyperDrive arguments into /tmp/amlt_run_hd.sh
|
| 13 |
+
2025-04-10 10:30:06,183:amlt-code-runner:INFO - Parsing tracking uri /mlflow/v1.0/subscriptions/bbd59374-b76d-4cb5-88b6-2be35debc7cf/resourceGroups/jordan/providers/Microsoft.MachineLearningServices/workspaces/jordanws
|
| 14 |
+
2025-04-10 10:30:06,183:amlt-code-runner:INFO - Tracking uri /mlflow/v1.0/subscriptions/bbd59374-b76d-4cb5-88b6-2be35debc7cf/resourceGroups/jordan/providers/Microsoft.MachineLearningServices/workspaces/jordanws has sub id bbd59374-b76d-4cb5-88b6-2be35debc7cf, resource group jordan, and workspace jordanws
|
| 15 |
+
2025-04-10 10:30:06,183:aml_token_auth:WARNING - The AzureMLTokenAuthentication created will not be updated due to missing params. The token expires on 2025-04-30 23:33:23.
|
| 16 |
|
| 17 |
+
2025-04-10 10:30:06,185:urllib3.connectionpool:DEBUG - Starting new HTTPS connection (1): eastus.api.azureml.ms:443
|
| 18 |
+
2025-04-10 10:30:06,695:urllib3.connectionpool:DEBUG - https://eastus.api.azureml.ms:443 "POST /mlflow/v1.0/subscriptions/bbd59374-b76d-4cb5-88b6-2be35debc7cf/resourceGroups/jordan/providers/Microsoft.MachineLearningServices/workspaces/jordanws/api/2.0/mlflow/runs/set-tag HTTP/11" 200 3
|
| 19 |
+
2025-04-10 10:30:06,719:amlt-code-runner:INFO - Executing ./amlt_setup.sh, /tmp/amlt_run_hd.sh
|
| 20 |
+
2025-04-10 10:30:06,789:background_dirsync:INFO - Starting directory syncer from '/scratch/amlt_code/outputs' to '/mnt/output/projects/amlt_project/amlt-results/sincere_feast_96p6v4qzfp_156', every 30.000000s
|
| 21 |
+
2025-04-10 10:30:06,793:background_dirsync:INFO - Starting directory syncer from '/scratch/azureml/cr/j/35fb2799c61349d98b8123fffa49511a/exe/wd/logs' to '/scratch/amlt_code/outputs/logs', every 30.000000s
|
model-00001-of-00003.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 4943162336
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5273d45e36ffc9209e24bf3e115e033d6fb8fe948cbe61e084ff9e57cd9e5523
|
| 3 |
size 4943162336
|
model-00002-of-00003.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 4999819336
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e27103ca8e40841e026bfe28c7d48062d3a0f98cf3895a280a8d8c28023219bb
|
| 3 |
size 4999819336
|
model-00003-of-00003.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 4540516344
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d97a8e78c397b970cb40fdc1760eee356adc500b64d6bfcc0ed7f4fa1024fc9
|
| 3 |
size 4540516344
|
training_args.bin
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 7736
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7fc6a513886ab8b2aca781148a20543f691866b50de131939a57ef261a093fca
|
| 3 |
size 7736
|