Merge pull request #85 from marian-nmt/master-updates

Use azcopy for downloading data if available

2 files changed, 25 insertions, 4 deletions

diff --git a/models/download-models.sh b/models/download-models.sh
index 88432c1..f3a7199 100644
--- a/models/download-models.sh
+++ b/models/download-models.sh
@@ -27,6 +27,12 @@ MODEL_TARBALLS=(
     #char-s2s       # A character-level RNN model (obsolete)
 )
 
+AZCOPY=true
+if ! command -v azcopy &> /dev/null; then
+    echo "Warning: 'azcopy' is not installed in your system. Downloading with 'wget'."
+    AZCOPY=false
+fi
+
 if [ $# -gt 0 ]; then
     echo The list of parameters is not empty.
     echo Skipping models not in the list: $*
@@ -43,7 +49,11 @@ for model in ${MODEL_TARBALLS[@]}; do
     fi
 
     echo Downloading checksum for $file ...
-    wget -nv -O- $URL/$file.md5 > $model.md5.newest
+    if $AZCOPY; then
+        azcopy copy "$URL/$file.md5" $model.md5.newest
+    else
+        wget -nv -O- $URL/$file.md5 > $model.md5.newest
+    fi
 
     # Do not download if the checksum files are identical, i.e. the archive has
     # not been updated since it was downloaded last time
@@ -51,7 +61,11 @@ for model in ${MODEL_TARBALLS[@]}; do
         echo File $file does not need to be updated
     else
         echo Downloading $file ...
-        wget -nv $URL/$file
+        if $AZCOPY; then
+            azcopy copy "$URL/$file" .
+        else
+            wget -nv $URL/$file
+        fi
         # Extract the archive
         tar zxf $file
         # Remove archive to save disk space
diff --git a/run_mrt.sh b/run_mrt.sh
index bdf2dee..3c3da4e 100755
--- a/run_mrt.sh
+++ b/run_mrt.sh
@@ -14,6 +14,8 @@
 #  - NUM_DEVICES - maximum number of GPU devices to be used
 #  - TIMEOUT - maximum duration for execution of a single test in the format
 #    accepted by the timeout command; set to 0 to disable
+#  - MODELS - path to the directory with models, default: ./models
+#  - DATA - path to the directory with data, default: ./data
 
 SHELL=/bin/bash
 
@@ -32,8 +34,13 @@ log "Running on $(hostname) as process $$"
 export MRT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export MRT_TOOLS=$MRT_ROOT/tools
 export MRT_MARIAN="$( realpath ${MARIAN:-$MRT_ROOT/../build} )"
-export MRT_MODELS=$MRT_ROOT/models
-export MRT_DATA=$MRT_ROOT/data
+
+# Print folders which contain models and data for regression tests
+export MRT_MODELS="$( realpath ${MODELS:-$MRT_ROOT/models} )"
+export MRT_DATA="$( realpath ${DATA:-$MRT_ROOT/data} )"
+
+log "Using models from: $MRT_MODELS"
+log "Using data from: $MRT_DATA"
 
 # Try adding build/ to MARIAN for backward compatibility
 if [[ ! -e $MRT_MARIAN/marian-decoder ]]; then