Commit 8dfdf186 authored by Yori Fournier's avatar Yori Fournier
Browse files

started run_step.sh and corresponding files

parent c52113ea
PWD=`pwd` 1. install docker, minikube
export PATH=$PATH:${PWD}/bin
\ No newline at end of file 2. mkdir pipeline
3. mkdir images
4. mkdir archives-pv (and populate it with the archive you need: step-name/all you need)
5. mkdir artifacts-pv
6. mkdir runtime-parameters-pv
6. start minikube: minikube start
7. mount permanent volume (in other terminal)
minikube mount archives-pv:/mnt/archives-pv
minikube mount artifacts-pv:/mnt/artifacts-pv
minikube mount runtime-parameters-pv:/mnt/parameters-pv
8. Create the permanent volume
kubectl apply -f create-archives-pv.yaml
kubectl apply -f create-artifacts-pv.yaml
kubectl apply -f create-parameters-pv.yaml
9. set config.toml as wanted
10. setup env
PWD=`pwd`
export PATH=$PATH:${PWD}/bin
11. setup-pipeline.sh (uses config.toml)
12. build_pipeline.sh (comming soon)
13. run_pipeline.sh (comming soon)
14. enjoy the results in artifacts-pv !!
STEP_NAME=$1
COMMIT=$2
source pipe-env.sh
# SETUP TMP DIR
TMP_HASH=$(date | sha1sum)
TMP_HASH=${TMP_HASH%% *}
echo ${TMP_HASH}
mkdir ${PIPE_ROOT}/${ARTIFACTS}/.${STEP_NAME}_${TMP_HASH}
TMP_DIR=${PIPE_ROOT}/${ARTIFACTS}/.${STEP_NAME}_${TMP_HASH}
touch ${TMP_DIR}/trace.toml
echo "[${STEP_NAME}]" >> ${TMP_DIR}/trace.toml
touch ${TMP_DIR}/parameters_checksum.sha1
# BUILD HASH
BUILD_HASH=$(docker inspect ${STEP_NAME}:${COMMIT} --format="{{.Id}}")
echo "build_hash = \"${BUILD_HASH}\"" >> ${TMP_DIR}/trace.toml
# PARAM HASH (only necessary if Parameters not in GIT repo --> COMMIT)
cd ${PIPE_ROOT}/${RUNTIME_PARAMS}
filenames=$(find "${STEP_NAME}" -type f)
for filename in ${filenames};
do
sha1sum ${filename} >> ${TMP_DIR}/parameters_checksum.sha1
done
cd ${PIPE_ROOT}
PARAM_HASH=$(sha1sum ${TMP_DIR}/parameters_checksum.sha1)
PARAM_HASH=${PARAM_HASH%% *}
echo "parameters_hash = \"sha1:${PARAM_HASH}\"" >> ${TMP_DIR}/trace.toml
# ARCHIVE HASH (only necessary if ArchDescr not in GIT repo --> COMMIT)
STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
echo $STEPS
if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
echo "ERROR: the step: ${STEP_NAME} is not defined."
exit 1
fi
ARCHIVE_DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.archives`)
for archive in "${ARCHIVE_DEPS[@]}";
do
cd ${PIPE_ROOT}/${ARCHIVES}
filenames=$(find "${archive}" -type f)
for filename in ${filenames};
do
sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
done
cd ${PIPE_ROOT}
done;
ARCHIVES_HASH=$(sha1sum ${TMP_DIR}/archives_checksum.sha1)
ARCHIVES_HASH=${ARCHIVES_HASH%% *}
echo "archives_hash = \"sha1:${ARCHIVES_HASH}\"" >> ${TMP_DIR}/trace.toml
# CREATE STEP HASH
STEP_HASH_ID="${BUILD_HASH}-${PARAM_HASH}-${ARCHIVES_HASH}"
STEP_HASH=$(echo "${STEP_HASH}" | sha1sum)
echo ${STEP_HASH_ID}
echo ${STEP_HASH}
echo "step-id = \"${STEP_HASH_ID}\"" >> ${TMP_DIR}/trace.toml
echo "step-hash = \"sha1:${STEP_HASH}\"" >> ${TMP_DIR}/trace.toml
exit 0
# CREATE DEPS HASH
STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
echo $STEPS
if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
echo "ERROR: the step: ${STEP_NAME} is not defined."
exit 1
fi
ARTIFACTS_DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.artifacts`)
for archive in "${ARTIFACTS_DEPS[@]}";
do
cd ${PIPE_ROOT}/${ARCHIVES}
filenames=$(find "${archive}" -type f)
for filename in ${filenames};
do
sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
done
cd ${PIPE_ROOT}
done;
exit 0
#PARAM_HASH=
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
PIPE_ROOT=`pwd` PIPE_ROOT=`pwd`
source ${PIPE_ROOT}/pipe-env.sh
source ${PIPE_ROOT}/src/setup.sh source ${PIPE_ROOT}/src/setup.sh
setup_pipeline setup_pipeline
......
...@@ -34,6 +34,7 @@ fi; ...@@ -34,6 +34,7 @@ fi;
PIPE_ROOT=`pwd` PIPE_ROOT=`pwd`
source ${PIPE_ROOT}/pipe-env.sh
source ${PIPE_ROOT}/src/setup.sh source ${PIPE_ROOT}/src/setup.sh
setup_step ${PIPE_ROOT} ${STEP_NAME} setup_step ${PIPE_ROOT} ${STEP_NAME}
......
...@@ -8,6 +8,8 @@ type = "git_repo" ...@@ -8,6 +8,8 @@ type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git" repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
commit = "1ccb3969" commit = "1ccb3969"
archives = ["ml_solar_plates_mock"]
[steps.label-features] [steps.label-features]
type = "git_repo" type = "git_repo"
......
export PIPE_ROOT=`pwd` export PIPE_ROOT=`pwd`
export IMAGES_ARCH="images" export IMAGES_ARCH="images"
export PIPELINE="pipeline" export PIPELINE="pipeline"
export RUNTIME_PARAMS="runtime-parameters-pv"
export ARTIFACTS="artifacts-pv"
export ARCHIVES="archives-pv"
...@@ -105,12 +105,26 @@ function setup_step() { ...@@ -105,12 +105,26 @@ function setup_step() {
local REPO_URL=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.repo_url` local REPO_URL=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.repo_url`
local COMMIT=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.commit` local COMMIT=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.commit`
# setup the step
setup_step_from_git_repo ${PIPE_ROOT} ${STEP_NAME} ${REPO_URL} ${COMMIT} setup_step_from_git_repo ${PIPE_ROOT} ${STEP_NAME} ${REPO_URL} ${COMMIT}
if [[ $? != 0 ]]; if [[ $? != 0 ]];
then then
return $? return $?
fi; fi;
# create a runtime-parameters for the step
if [ ! -d ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME} ];
then
mkdir ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME}
if [[ $? != 0 ]];
then
echo "ERROR: Could not create the runtime-parameters folder. Do you have the write-rights there: ${PIPE_ROOT}/${RUNTIME_PARAM} ?"
return $?
fi;
fi;
;; ;;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment