Commit 14f9c08d authored by Yori Fournier's avatar Yori Fournier
Browse files

add setup.sh as source and now use it for setup_pipeline.sh and setup_step.sh

parent 8befc741
# READ pipeline config
function checkout_commit() {
# args:
# PIPE_ROOT: The root path of the pipeline
# STEP_NAME: The name of the step to setup
# COMMIT: The commit to be used
PIPE_ROOT=$1
STEP_NAME=$2
COMMIT=$4
local STEPS_DIR=${PIPE_ROOT}/pipeline
local STEP_DIR=${STEPS_DIR}/${STEP_NAME}
cd ${STEP_DIR}
git checkout ${COMMIT} > /dev/null 2>&1
CUR_COMMIT=`git rev-parse --short HEAD`
echo "The repo is now in commit: ${CUR_COMMIT}"
if [[ $? != 0 ]];
then
echo "ERROR: could not checkout the provided commit"
exit 1
fi;
cd ${PIPE_ROOT}
}
function setup_step_from_git_repo() {
# args:
# PIPE_ROOT: The root path of the pipeline
# STEP_NAME: The name of the step to setup
# REPO_URL: The https url of the repo (must be public)
# COMMIT: The commit to be used
PIPE_ROOT=$1
STEP_NAME=$2
REPO_URL=$3
COMMIT=$4
local STEPS_DIR="${PIPE_ROOT}/pipeline"
local STEP_DIR="${STEPS_DIR}/${STEP_NAME}"
echo ">>> Setup ${STEP_NAME} from ${REPO_URL}:${COMMIT}"
# If the step does not exists in the pipeline
if [ ! -d ${STEP_DIR} ];
then
cd ${STEPS_DIR}
git clone ${REPO_URL}
cd ${PIPE_ROOT}
fi;
# if git repo then Checkout commit
if [ -d ${STEP_DIR}/.git ];
then
checkout_commit ${PIPE_ROOT} ${STEP_NAME} ${COMMIT}
else
echo "ERROR: the source must be under Git version control"
fi;
}
#!/bin/env bash
PIPE_ROOT=`pwd`
STEP_NAMES=(`./stoml ${PIPE_ROOT}/config.toml pipeline`)
echo ">>> Steps: ${STEP_NAMES[@]}"
for STEP_NAME in "${STEP_NAMES[@]}";
do
TYPE=`./stoml ${PIPE_ROOT}/config.toml ${STEP_NAME}.type`
case $TYPE in
git_repo)
REPO_URL=`./stoml ${PIPE_ROOT}/config.toml ${STEP_NAME}.repo_url`
COMMIT=`./stoml ${PIPE_ROOT}/config.toml ${STEP_NAME}.commit`
setup_step_from_git_repo ${PIPE_ROOT} ${STEP_NAME} ${REPO_URL} ${COMMIT}
;;
*)
echo "TYPE: ${TYPE} not supported."
;;
esac
done;
exit 0
source ${PIPE_ROOT}/src/setup.sh
setup_pipeline
exit $?
#!/bin/env bash
function usage() {
echo "Usage: "
echo " setup_step.sh -h/--help"
echo " setup_step.sh <STEP-NAME>"
}
# Process command line options
shortopts="h"
longopts="help"
opts=$(getopt -o "$shortopts" -l "$longopts" -n "$(basename $0)" -- "$@")
if [ $? -eq 0 ]; then
#eval set -- ${opts}
eval "set -- ${opts}"
while [ $# -gt 0 ]; do
case "$1" in
( -- ) shift; break;;
( -h|--help ) usage; exit 2; shift; break;;
esac
done
else
usage; exit 2;
fi
STEP_NAME=$1
if [[ "${STEP_NAME}" == "" ]];
then
echo "STEP: ${STEP_NAME}"
usage; exit 2;
fi;
PIPE_ROOT=`pwd`
source ${PIPE_ROOT}/src/setup.sh
setup_step ${PIPE_ROOT} ${STEP_NAME}
exit $?
pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-features", "extract-features-properties"]
[extract-features]
[steps]
[steps.extract-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
commit = "1ccb3969"
[label-features]
[steps.label-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
commit = "a4ce45b2"
[select-meaningful-plates-and-features]
[steps.select-meaningful-plates-and-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
commit = "5fc5a7bc"
[extract-features-properties]
[steps.extract-features-properties]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features-properties.git"
......
#!/bin/env bash
function checkout_commit() {
# args:
# PIPE_ROOT: The root path of the pipeline
# STEP_NAME: The name of the step to setup
# COMMIT: The commit to be used
local PIPE_ROOT=$1
local STEP_NAME=$2
local COMMIT=$4
local STEPS_DIR=${PIPE_ROOT}/pipeline
local STEP_DIR=${STEPS_DIR}/${STEP_NAME}
cd ${STEP_DIR}
git checkout ${COMMIT} > /dev/null 2>&1
if [[ $? != 0 ]];
then
echo "ERROR: could not checkout the provided commit"
return $?
fi;
local CUR_COMMIT=`git rev-parse --short HEAD`
echo "The repo is now in commit: ${CUR_COMMIT}"
cd ${PIPE_ROOT}
return 0
}
function setup_step_from_git_repo() {
# args:
# PIPE_ROOT: The root path of the pipeline
# STEP_NAME: The name of the step to setup
# REPO_URL: The https url of the repo (must be public)
# COMMIT: The commit to be used
local PIPE_ROOT=$1
local STEP_NAME=$2
local REPO_URL=$3
local COMMIT=$4
local STEPS_DIR="${PIPE_ROOT}/pipeline"
local STEP_DIR="${STEPS_DIR}/${STEP_NAME}"
echo ">>> Setup ${STEP_NAME} from ${REPO_URL}:${COMMIT}"
# If the step does not exists in the pipeline
if [ ! -d ${STEP_DIR} ];
then
cd ${STEPS_DIR}
git clone ${REPO_URL}
if [[ $? != 0 ]];
then
echo "ERROR: could not git clone from the provided url"
return $?
fi;
cd ${PIPE_ROOT}
fi;
# if git repo then Checkout commit
if [ -d ${STEP_DIR}/.git ];
then
checkout_commit ${PIPE_ROOT} ${STEP_NAME} ${COMMIT}
if [[ $? != 0 ]];
then
return $?
fi;
else
echo "ERROR: the source must be under Git version control"
return 1
fi;
return 0
}
function setup_step() {
# args:
# PIPE_ROOT: The root path of the pipeline
# STEP_NAME: The name of the step to setup
local PIPE_ROOT=$1
local STEP_NAME=$2
local STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
echo "ERROR: the step: ${STEP_NAME} is not defined."
return 1
fi
local TYPE=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.type`
case $TYPE in
git_repo)
local REPO_URL=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.repo_url`
local COMMIT=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.commit`
setup_step_from_git_repo ${PIPE_ROOT} ${STEP_NAME} ${REPO_URL} ${COMMIT}
if [[ $? != 0 ]];
then
return $?
fi;
;;
*)
echo "ERROR: TYPE: ${TYPE} not supported."
return 1
;;
esac
return 0
}
function setup_pipeline() {
local PIPE_ROOT=`pwd`
local STEP_NAMES=(`./stoml ${PIPE_ROOT}/config.toml pipeline`)
echo ">>> Steps: ${STEP_NAMES[@]}"
for STEP_NAME in "${STEP_NAMES[@]}";
do
setup_step ${PIPE_ROOT} ${STEP_NAME}
if [[ $? != 0 ]];
then
echo "ERROR: Could not setup step: ${STEP_NAME}"
return $?
fi;
done;
return 0
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment