New commands init and store-credentials

parent 6d052c97
...@@ -9,6 +9,7 @@ DATASET_REPORT_FILE_NAME="/.dataset-task-report" ...@@ -9,6 +9,7 @@ DATASET_REPORT_FILE_NAME="/.dataset-task-report"
DATASET_COMPLETE_FILE_NAME="/.dataset-completed" DATASET_COMPLETE_FILE_NAME="/.dataset-completed"
DISCARD_CHANGES_FILE_NAME="/.discard-changes" DISCARD_CHANGES_FILE_NAME="/.discard-changes"
LOG_DIR="$EBULK_DATA_PATH/logs" LOG_DIR="$EBULK_DATA_PATH/logs"
CREDENTIALS_FILE="$EBULK_DATA_PATH/.credentials"
TOOL_PATH="$(dirname "$0")/ebulk-data" TOOL_PATH="$(dirname "$0")/ebulk-data"
DOWN_FILE="$EBULK_DATA_PATH/download-config.yml" DOWN_FILE="$EBULK_DATA_PATH/download-config.yml"
DOWN_TEMPLATE_FILE="$TOOL_PATH/config/download-config_template.yml" DOWN_TEMPLATE_FILE="$TOOL_PATH/config/download-config_template.yml"
...@@ -125,11 +126,11 @@ function checkParameters { ...@@ -125,11 +126,11 @@ function checkParameters {
fi fi
} }
function askCredentials { function storeCredentials {
echo echo
echo "Please, enter your ebulk user and password:" echo "Please, enter your ebulk user and password:"
echo echo
echo "** You can use the same Telecom-Wendelin-IA site user and password **" echo "** You can use your Telecom-Wendelin-IA credentials **"
echo "** If you don't have a user, please feel free to request one to roqueporchetto@gmail.com **" echo "** If you don't have a user, please feel free to request one to roqueporchetto@gmail.com **"
echo echo
echo "User:" echo "User:"
...@@ -142,6 +143,8 @@ function askCredentials { ...@@ -142,6 +143,8 @@ function askCredentials {
echo >&2; return 1 echo >&2; return 1
fi fi
PASS= read -s -e -p Password: pwd PASS= read -s -e -p Password: pwd
echo
echo "$USER;$pwd" > "$CREDENTIALS_FILE" 2>/dev/null
} }
function updateConfigFile { function updateConfigFile {
...@@ -176,10 +179,9 @@ function updateConfigFile { ...@@ -176,10 +179,9 @@ function updateConfigFile {
$PARAMETER_FUNCTION $PARAMETER_FUNCTION
fi fi
TOOL_DIR=\"$LOG_DIR\" TOOL_DIR=\"$EBULK_DATA_PATH\"
DATA_SET=\"$DATA_SET\" DATA_SET=\"$DATA_SET\"
USER=\"$USER\" USER=\"$USER\"
pwd=\"$pwd\"
CHUNK=\"$CHUNK\" CHUNK=\"$CHUNK\"
DATASET_DIR=\"$DATASET_DIR\" DATASET_DIR=\"$DATASET_DIR\"
DOWN_URL=\"$DOWN_URL\" DOWN_URL=\"$DOWN_URL\"
...@@ -226,12 +228,6 @@ function runProcess { ...@@ -226,12 +228,6 @@ function runProcess {
fi fi
fi fi
fi fi
if [ -z "$STATUS" ]; then
if ! askCredentials; then
return 1
fi
fi
echo
updateConfigFile updateConfigFile
echo "[INFO] Starting operation..." echo "[INFO] Starting operation..."
if [ ! -d $LOG_DIR ]; then if [ ! -d $LOG_DIR ]; then
...@@ -516,7 +512,10 @@ while [ "$1" != "" ]; do ...@@ -516,7 +512,10 @@ while [ "$1" != "" ]; do
-r | --readme ) less $TOOL_PATH/README.md -r | --readme ) less $TOOL_PATH/README.md
exit exit
;; ;;
status | push | pull ) OPERATION=$1 store-credentials ) storeCredentials
exit
;;
status | push | pull | init ) OPERATION=$1
;; ;;
add | remove | reset ) OPERATION=$1 add | remove | reset ) OPERATION=$1
shift shift
...@@ -586,6 +585,28 @@ case $OPERATION in ...@@ -586,6 +585,28 @@ case $OPERATION in
echo echo
runProcess runProcess
;; ;;
init)
EBULK_DATASET_FILE="$DATASET_DIR$EBULK_DATASET_FILE_NAME"
if [ -f "$EBULK_DATASET_FILE" ]; then
echo
echo -e "${ORANGE}[WARNING] The specified directory was already init as data set."
echo -e "[WARNING] Do you want to reset this directory as dataset? (Y/n)${NC}"
read -e OPTION
if [ "$OPTION" = "n" ] ; then
exit
fi
fi
checkParameters
if [ ! $? -eq 0 ]; then
exit
fi
DATASET_REPORT_FILE="$DATASET_DIR$DATASET_REPORT_FILE_NAME"
DATASET_COMPLETE_FILE="$DATASET_DIR$DATASET_COMPLETE_FILE_NAME"
rm $DATASET_REPORT_FILE 2>/dev/null
rm $DATASET_COMPLETE_FILE 2>/dev/null
touch $DATASET_REPORT_FILE 2>/dev/null
touch $DATASET_COMPLETE_FILE 2>/dev/null
;;
pull) pull)
welcome welcome
FILE=$DOWN_FILE FILE=$DOWN_FILE
......
...@@ -4,8 +4,6 @@ exec: ...@@ -4,8 +4,6 @@ exec:
in: in:
type: wendelin type: wendelin
erp5_url: $DOWN_URL erp5_url: $DOWN_URL
user: $USER
password: $pwd
data_set: $DATA_SET data_set: $DATA_SET
chunk_size: $CHUNK chunk_size: $CHUNK
output_path: $DATASET_DIR output_path: $DATASET_DIR
......
...@@ -8,16 +8,13 @@ in: ...@@ -8,16 +8,13 @@ in:
data_set: $DATA_SET data_set: $DATA_SET
chunk_size: $CHUNK chunk_size: $CHUNK
erp5_url: $DOWN_URL erp5_url: $DOWN_URL
user: $USER
password: $pwd
tool_dir: $TOOL_DIR tool_dir: $TOOL_DIR
status: $STATUS status: $STATUS
out: out:
type: wendelin type: wendelin
erp5_url: $ING_URL erp5_url: $ING_URL
user: $USER
password: $pwd
type_input: "filesystem" type_input: "filesystem"
data_set: $DATA_SET data_set: $DATA_SET
erp5_base_url: $DOWN_URL erp5_base_url: $DOWN_URL
tool_dir: $TOOL_DIR
...@@ -28,8 +28,7 @@ in: ...@@ -28,8 +28,7 @@ in:
out: out:
type: wendelin type: wendelin
erp5_url: $ING_URL erp5_url: $ING_URL
user: $USER tool_dir: $TOOL_DIR
password: $pwd
data_set: $DATA_SET data_set: $DATA_SET
erp5_base_url: $DOWN_URL erp5_base_url: $DOWN_URL
......
...@@ -23,8 +23,7 @@ in: ...@@ -23,8 +23,7 @@ in:
out: out:
type: wendelin type: wendelin
erp5_url: $ING_URL erp5_url: $ING_URL
user: $USER tool_dir: $TOOL_DIR
password: $pwd
data_set: $DATA_SET data_set: $DATA_SET
erp5_base_url: $DOWN_URL erp5_base_url: $DOWN_URL
......
...@@ -25,8 +25,7 @@ in: ...@@ -25,8 +25,7 @@ in:
out: out:
type: wendelin type: wendelin
erp5_url: $ING_URL erp5_url: $ING_URL
user: $USER tool_dir: $TOOL_DIR
password: $pwd
data_set: $DATA_SET data_set: $DATA_SET
erp5_base_url: $DOWN_URL erp5_base_url: $DOWN_URL
......
...@@ -29,8 +29,7 @@ in: ...@@ -29,8 +29,7 @@ in:
out: out:
type: wendelin type: wendelin
erp5_url: $ING_URL erp5_url: $ING_URL
user: $USER tool_dir: $TOOL_DIR
password: $pwd
data_set: $DATA_SET data_set: $DATA_SET
erp5_base_url: $DOWN_URL erp5_base_url: $DOWN_URL
......
...@@ -15,6 +15,7 @@ class DatasetUtils ...@@ -15,6 +15,7 @@ class DatasetUtils
SPLIT_FILE = ".split-operation" SPLIT_FILE = ".split-operation"
SPLIT_CONTROL_FILE = ".control-split-operation" SPLIT_CONTROL_FILE = ".control-split-operation"
FIRST_INGESTION_FILE = ".first-ingestion" FIRST_INGESTION_FILE = ".first-ingestion"
CREDENTIALS_FILE = ".credentials"
RUN_DONE = "done" RUN_DONE = "done"
RUN_ERROR = "error" RUN_ERROR = "error"
...@@ -70,6 +71,41 @@ class DatasetUtils ...@@ -70,6 +71,41 @@ class DatasetUtils
}.flatten.select{ |file| File.file?(file) } }.flatten.select{ |file| File.file?(file) }
end end
def getCredentials(tool_dir)
credential_path = appendSlashTo(tool_dir) + CREDENTIALS_FILE
if File.exist?(credential_path)
credentials = File.open(credential_path).read.chomp.split(RECORD_SEPARATOR)
user = credentials[0]
password = credentials[1]
@logger.info("Using stored credentials for user '#{user}'", print=TRUE)
else
puts
puts "Please, enter your ebulk user and password:"
puts
puts "** You can use your Telecom-Wendelin-IA credentials **"
puts "** If you don't have a user, please feel free to request one to roqueporchetto@gmail.com **"
puts
@logger.info("Remember that you can store your credentials for automatic authentication by running 'ebulk store-credentials'", print=TRUE)
puts
print "User:"
user = gets.chomp
if not /^[A-Za-z][-A-Za-z0-9_]*$/.match(user)
puts
@logger.error("Invalid user name. Did enter an invalid character?", print=TRUE)
@logger.info("Please enter a valid user name.", print=TRUE)
@logger.abortExecution(error=FALSE)
end
print "Password:"
password = STDIN.noecho(&:gets).chomp
puts
puts
@logger.info("Remember that you can store your credentials for automatic authentication by running 'ebulk store-credentials'", print=TRUE)
puts
sleep 1
end
return user, password
end
def getLocalFiles(remove=nil) def getLocalFiles(remove=nil)
local_files = {} local_files = {}
begin begin
......
...@@ -14,7 +14,7 @@ class LogManager ...@@ -14,7 +14,7 @@ class LogManager
end end
def setFilename(tool_dir, prefix) def setFilename(tool_dir, prefix)
log_dir = "#{tool_dir}/" log_dir = "#{tool_dir}/logs/"
if not File.directory?(log_dir) if not File.directory?(log_dir)
Dir.mkdir log_dir Dir.mkdir log_dir
end end
......
...@@ -79,7 +79,7 @@ module Embulk ...@@ -79,7 +79,7 @@ module Embulk
@logger = LogManager.instance() @logger = LogManager.instance()
@logger.setFilename(tool_dir, "ingestion") @logger.setFilename(tool_dir, "ingestion")
task = { 'paths' => [] } task = { 'paths' => [] }
task['supplier'] = config.param('supplier', :string) task['supplier'] = config.param('supplier', :string, default: "default")
task['data_set'] = config.param('data_set', :string) task['data_set'] = config.param('data_set', :string)
task['chunk_size'] = config.param('chunk_size', :float, default: 0) * DatasetUtils::MEGA task['chunk_size'] = config.param('chunk_size', :float, default: 0) * DatasetUtils::MEGA
if task['chunk_size'] == 0 if task['chunk_size'] == 0
...@@ -108,14 +108,16 @@ module Embulk ...@@ -108,14 +108,16 @@ module Embulk
end end
end end
end end
if not @status
user, password = @dataset_utils.getCredentials(tool_dir)
@wendelin = WendelinClient.new(config.param('erp5_url', :string), user, password)
end
@logger.info("Checking local files...", print=TRUE) @logger.info("Checking local files...", print=TRUE)
task['paths'] = @dataset_utils.getLocalPaths(paths) task['paths'] = @dataset_utils.getLocalPaths(paths)
if @status if @status
self.status(task) self.status(task)
@logger.abortExecution(error=FALSE) @logger.abortExecution(error=FALSE)
end end
@wendelin = WendelinClient.new(config.param('erp5_url', :string), config.param('user', :string), config.param('password', :string))
@logger.info("Checking remote dataset...", print=TRUE) @logger.info("Checking remote dataset...", print=TRUE)
data_stream_dict = @wendelin.getDataStreams(task['data_set']) data_stream_dict = @wendelin.getDataStreams(task['data_set'])
if data_stream_dict["status_code"] != 0 if data_stream_dict["status_code"] != 0
......
...@@ -87,21 +87,15 @@ module Embulk ...@@ -87,21 +87,15 @@ module Embulk
@erp5_url = config.param('erp5_url', :string) @erp5_url = config.param('erp5_url', :string)
@data_set = config.param('data_set', :string) @data_set = config.param('data_set', :string)
@logger.info("Dataset name: #{@data_set}") @logger.info("Dataset name: #{@data_set}")
@user = config.param("user", :string, defualt: nil)
@logger.info("User: #{@user}")
@password = config.param("password", :string, default: nil)
@chunk_size = config.param('chunk_size', :float, default: 0) * DatasetUtils::MEGA @chunk_size = config.param('chunk_size', :float, default: 0) * DatasetUtils::MEGA
@output_path = config.param("output_path", :string, :default => nil) @output_path = config.param("output_path", :string, :default => nil)
if not File.directory?(@output_path) if not File.directory?(@output_path)
@logger.error("Output directory not found.", print=TRUE) @logger.error("Output directory not found.", print=TRUE)
@logger.abortExecution() @logger.abortExecution()
end end
@wendelin = WendelinClient.new(@erp5_url, @user, @password)
task = { task = {
'erp5_url' => @erp5_url, 'erp5_url' => @erp5_url,
'data_set' => @data_set, 'data_set' => @data_set,
'user' => @user,
'password' => @password,
'chunk_size' => @chunk_size, 'chunk_size' => @chunk_size,
'output_path' => @output_path, 'output_path' => @output_path,
'tool_dir' => @tool_dir 'tool_dir' => @tool_dir
...@@ -125,6 +119,8 @@ module Embulk ...@@ -125,6 +119,8 @@ module Embulk
@logger.abortExecution(error=FALSE) @logger.abortExecution(error=FALSE)
end end
end end
task['user'], task['password'] = @dataset_utils.getCredentials(@tool_dir)
@wendelin = WendelinClient.new(@erp5_url, task['user'], task['password'])
@logger.info("Getting remote file list from dataset '#{@data_set}'...", print=TRUE) @logger.info("Getting remote file list from dataset '#{@data_set}'...", print=TRUE)
data_stream_list = @wendelin.getDataStreams(@data_set) data_stream_list = @wendelin.getDataStreams(@data_set)
if data_stream_list["status_code"] == 0 if data_stream_list["status_code"] == 0
......
...@@ -11,18 +11,18 @@ module Embulk ...@@ -11,18 +11,18 @@ module Embulk
def self.transaction(config, schema, count, &control) def self.transaction(config, schema, count, &control)
task = { task = {
"erp5_url" => config.param("erp5_url", :string), "erp5_url" => config.param("erp5_url", :string),
"user" => config.param("user", :string, defualt: nil),
"password" => config.param("password", :string, default: nil),
"path_prefix" => config.param("path_prefix", :string, :default => nil), "path_prefix" => config.param("path_prefix", :string, :default => nil),
"type_input" => config.param("type_input", :string, :default => nil), "type_input" => config.param("type_input", :string, :default => nil),
"data_set" => config.param("data_set", :string, default: nil), "data_set" => config.param("data_set", :string, default: nil),
"erp5_base_url" => config.param("erp5_base_url", :string, default: nil) "erp5_base_url" => config.param("erp5_base_url", :string, default: nil),
"tool_dir" => config.param('tool_dir', :string)
} }
storage_ingestion = ! task["type_input"] storage_ingestion = ! task["type_input"]
@dataset_utils = DatasetUtils.new(Dir.pwd)
task["user"], task["password"] = @dataset_utils.getCredentials(task["tool_dir"])
task_reports = yield(task) task_reports = yield(task)
next_config_diff = {} next_config_diff = {}
@logger = LogManager.instance() @logger = LogManager.instance()
@dataset_utils = DatasetUtils.new(Dir.pwd)
if task_reports.length > 0 if task_reports.length > 0
@logger.info("Your ingested files will be available in the site in a few minutes. Thank for your patience.", print=TRUE) @logger.info("Your ingested files will be available in the site in a few minutes. Thank for your patience.", print=TRUE)
if storage_ingestion if storage_ingestion
...@@ -50,10 +50,8 @@ module Embulk ...@@ -50,10 +50,8 @@ module Embulk
def init def init
credentials = {} credentials = {}
@erp5_url = task["erp5_url"] @erp5_url = task["erp5_url"]
@user = task["user"]
@password = task["password"]
@logger = LogManager.instance() @logger = LogManager.instance()
@wendelin = WendelinClient.new(@erp5_url, @user, @password) @wendelin = WendelinClient.new(@erp5_url, task["user"], task["password"])
end end
def add(page) def add(page)
......
...@@ -93,7 +93,9 @@ module Embulk ...@@ -93,7 +93,9 @@ module Embulk
@dataset_utils.deleteSplitOperationControlFile(reference) @dataset_utils.deleteSplitOperationControlFile(reference)
@logger.abortExecution() @logger.abortExecution()
rescue Exception => e rescue Exception => e
puts
@logger.error("An error occurred during file ingestion: " + e.to_s, print=TRUE) @logger.error("An error occurred during file ingestion: " + e.to_s, print=TRUE)
puts
@logger.error(e.backtrace) @logger.error(e.backtrace)
puts "[INFO] For more detailed information, please refer to the log file: " + @logger.getLogPath() puts "[INFO] For more detailed information, please refer to the log file: " + @logger.getLogPath()
@dataset_utils.deleteSplitOperationControlFile(reference) @dataset_utils.deleteSplitOperationControlFile(reference)
......
...@@ -8,6 +8,7 @@ require_relative 'filelogger' ...@@ -8,6 +8,7 @@ require_relative 'filelogger'
class WendelinClient class WendelinClient
HTTP_MESSAGE_401 = "Unauthorized access. Please check your user credentials and try again." HTTP_MESSAGE_401 = "Unauthorized access. Please check your user credentials and try again."
HTTP_MESSAGE_400 = "There was a problem with the http request. If the error persists, please contact the administrator."
HTTP_MESSAGE_5XX = "Internal Server Error: if the error persists, please contact the administrator." HTTP_MESSAGE_5XX = "Internal Server Error: if the error persists, please contact the administrator."
HTTP_MESSAGE_OTHER = "Sorry, an error ocurred. If the error persists, please contact the administrator." HTTP_MESSAGE_OTHER = "Sorry, an error ocurred. If the error persists, please contact the administrator."
HTTP_MESSAGE_NOT_2XX = "HTTP-NOT-OK" HTTP_MESSAGE_NOT_2XX = "HTTP-NOT-OK"
......
...@@ -9,12 +9,14 @@ commands: ...@@ -9,12 +9,14 @@ commands:
pull [<dataset>] Downloads the content of the target dataset from the site into the output location pull [<dataset>] Downloads the content of the target dataset from the site into the output location
push [<dataset>] Ingests the content of the input location into a target dataset on the site push [<dataset>] Ingests the content of the input location into a target dataset on the site
status [<dataset>] Lists the local changes of target dataset status [<dataset>] Lists the local changes of target dataset
init [<dataset>] Sets the directory as a dataset, so add and remove operations can be performed
add <path> Marks new or modified files in path for ingestion add <path> Marks new or modified files in path for ingestion
remove <path> Marks files in path for removal remove <path> Marks files in path for removal
reset <path> Resets marked files in path reset <path> Resets marked files in path
-h, --help Tool help -h, --help Tool help
-r, --readme Opens README file -r, --readme Opens README file
-e, --examples Shows some tool usage examples -e, --examples Shows some tool usage examples
store-credentials Stores user and password for automatic authentication
argument: argument:
dataset argument Unique reference for the target dataset dataset argument Unique reference for the target dataset
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment