New parameter 'batch size' for downloads

ebulk

@@ -6,7 +6,7 @@ DOWN_URL="$DEFAULT_DATA_LAKE_URL"
 ING_POLICY="portal_ingestion_policies/default_ebulk"
 ING_URL="$DEFAULT_DATA_LAKE_URL$ING_POLICY"
 
-EBULK_VERSION="1.1.6"
+EBULK_VERSION="1.1.7"
 EMBULK_VERSION="0.9.7"
 EBULK_DATA_PATH=~/.ebulk
 EBULK_DATASET_FILE_NAME="/.ebulk_dataset"
@@ -37,6 +37,7 @@ GREEN='\033[0;32m'
 ORANGE='\033[0;33m'
 NC='\033[0m'
 DEFAULT_CHUNK_SIZE="50"
+DEFAULT_BATCH_SIZE="1000"
 STAGE_ADD="add"
 STAGE_REMOVE="remove"
 STAGE_RESET="reset"
@@ -162,6 +163,15 @@ function checkParameters {
 	    helpReadme >&2; return 1
         fi
     fi
+    if [ ! -z "$BATCH_SIZE" ]; then
+        re='^[0-9]+$'
+        if ! [[ $BATCH_SIZE =~ $re ]] ; then
+            echo
+            echo -e "${ORANGE}[ERROR] Error in argument: batch size must be an integer.${NC}"
+            echo
+	    helpReadme >&2; return 1
+        fi
+    fi
 }
 
 function configure {
@@ -289,6 +299,7 @@ function updateConfigFile {
     DATA_SET=\"$DATA_SET\"
     USER=\"$USER\"
     CHUNK=\"$CHUNK\"
+    BATCH_SIZE=\"$BATCH_SIZE\"
     DATASET_DIR=\"$DATASET_DIR\"
     DOWN_URL=\"$DOWN_URL\"
     ING_URL=\"$ING_URL\"
@@ -333,6 +344,11 @@ function runProcess {
 	    else
 	    	echo "[INFO] Chunk size set in $CHUNK MB."
 	    fi
+	    if [ "$BATCH_SIZE" -eq "0" ]; then
+	      echo "[INFO] Default batch size: $DEFAULT_BATCH_SIZE MB."
+	    else
+	      echo "[INFO] Batch size set in $BATCH_SIZE files."
+	    fi
     	fi
 	if [ "$DATASET_DESCRIPTION" != "" ] ; then
 	  echo
@@ -637,6 +653,9 @@ while [ "$1" != "" ]; do
 	-c | --chunk )          	shift
 					CHUNK=$1
 					;;
+	-bs | --batch-size )     	shift
+					BATCH_SIZE=$1
+					;;
 	-h | --help )           	cat $TOOL_PATH/help.md
 					exit
 					;;
@@ -678,7 +697,7 @@ while [ "$1" != "" ]; do
     shift
 done
 
-for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk' '-dc' '--discard-changes' '-dd' '--set-description'; do
+for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk' '-bs' '--batch-size' '-dc' '--discard-changes' '-dd' '--set-description'; do
   if [ "$ELEMENT" = "$REFERENCE" ]; then
 	REFERENCE="."
   fi
@@ -696,6 +715,9 @@ fi
 if [[ $CHUNK = "" ]]; then
 	CHUNK=$DEFAULT_CHUNK_SIZE
 fi
+if [[ $BATCH_SIZE = "" ]]; then
+	BATCH_SIZE=$DEFAULT_BATCH_SIZE
+fi
 
 case $OPERATION in
     add)

ebulk-data/config/download-config_template.yml

@@ -6,6 +6,7 @@ in:
   erp5_url: $DOWN_URL
   data_set: $DATA_SET
   chunk_size: $CHUNK
+  batch_size: $BATCH_SIZE
   output_path: $DATASET_DIR
   tool_dir: $TOOL_DIR
   always_yes: $ALWAYS_YES