new command to set data lake url

ebulk

 #! /usr/bin/env bash
 
-DOWN_URL='https://softinst104003.lhost.vifib.net/erp5/'
-ING_URL='https://softinst104003.lhost.vifib.net/erp5/portal_ingestion_policies/wendelin_embulk'
+DATA_LAKE_URL='https://softinst104003.lhost.vifib.net/erp5/'
+
+DOWN_URL="$DATA_LAKE_URL/"
+ING_URL="$DATA_LAKE_URL/portal_ingestion_policies/wendelin_embulk"
 
 EBULK_VERSION="0.9.7"
 EBULK_DATA_PATH=~/.ebulk
@@ -12,6 +14,7 @@ DISCARD_CHANGES_FILE_NAME="/.discard-changes"
 LOG_DIR="$EBULK_DATA_PATH/logs"
 STATUS_FILE="$EBULK_DATA_PATH/.status"
 CREDENTIALS_FILE="$EBULK_DATA_PATH/.credentials"
+DATA_LAKE_URL_FILE="$EBULK_DATA_PATH/.data-lake-url"
 CONFIG_FILE="$EBULK_DATA_PATH/.config"
 DESCRIPTION_DATASET_PATH_PREFIX="$EBULK_DATA_PATH/.dataset-description"
 DESCRIPTION_DATASET_PENDING_PATH_PREFIX="$EBULK_DATA_PATH/.dataset-description-pending"
@@ -40,6 +43,15 @@ RESUME="R"
 DOWNLOAD="D"
 ASK="A"
 
+# load data lake url from file if exists
+if [ -f "$DATA_LAKE_URL_FILE" ]; then
+  URL=$(cat "$DATA_LAKE_URL_FILE" 2>/dev/null)
+  if [[ "$URL" != "" ]]; then
+    DOWN_URL="$URL/"
+    ING_URL="$URL/portal_ingestion_policies/wendelin_embulk"
+  fi
+fi
+
 function helpReadme {
     echo -e "[INFO] For help, please run '${GREEN}ebulk --help${NC}'"
     echo -e "[INFO] For README, please run '${GREEN}ebulk --readme${NC}'"
@@ -194,6 +206,28 @@ function storeCredentials {
     echo "$USER;$pwd" > "$CREDENTIALS_FILE" 2>/dev/null
 }
 
+function setDataLakeUrl {
+    echo
+    echo "Please, enter your data lake url:"
+    echo
+    echo "URL:"
+    read -e URL
+    re='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]'
+    if ! [[ $URL =~ $re ]] ; then
+      echo
+      echo -e "${ORANGE}[ERROR] Invalid url. Please make sure to enter a full url. e.g. https://my_data_lake.org/erp5"
+      echo -e "[INFO] Please enter a valid url.${NC}"
+      echo >&2; return 1
+    fi
+    echo "$URL" > "$DATA_LAKE_URL_FILE" 2>/dev/null
+}
+
+function defaultDataLakeUrl {
+  echo "" > "$DATA_LAKE_URL_FILE" 2>/dev/null
+  DOWN_URL="$DATA_LAKE_URL/"
+  ING_URL="$DATA_LAKE_URL/portal_ingestion_policies/wendelin_embulk"
+}
+
 function updateConfigFile {
     if [ "$STORAGE" != "" ] ; then
     	echo
@@ -584,6 +618,12 @@ while [ "$1" != "" ]; do
 	store-credentials )           	storeCredentials
 					exit
 					;;
+	set-data-lake-url )           	setDataLakeUrl
+					exit
+					;;
+	default-data-lake-url )         defaultDataLakeUrl
+					exit
+					;;
 	configure )           		configure
 					exit
 					;;

ebulk-data/embulk-wendelin-dataset-tool/lib/embulk/wendelin_client.rb

@@ -223,6 +223,7 @@ class WendelinClient
     rescue Exception => e
       puts
       @logger.error("HTTP ERROR: " + e.to_s, print=TRUE)
+      @logger.error("During request to " + uri.hostname.to_s, print=TRUE)
       @logger.error(e.backtrace)
       if e.to_s.include? "handshake_failure" or e.to_s.include? "protocol_version"
         @logger.error("This issue could be due to a cipher or certificates problem, please check your frontend instance or try with a http url", print=TRUE)

ebulk-data/help.md

@@ -17,6 +17,8 @@ commands:
    -r,  --readme         Opens README file
    -e,  --examples       Shows some tool usage examples
    store-credentials	 Stores user and password for automatic authentication
+   set-data-lake-url	 Sets the data lake url where to ingest/download
+   default-data-lake-url Sets the data lake url to default
    config		 Allows user to set tool automatic actions
 
 argument:
@@ -38,4 +40,3 @@ options:
    -a,  --advanced	  	  Allows to edit the Embulk cofiguration file of the input storage
    -dc, --discard-changes	  Discards local changes by checking the remote dataset
 
-