Commit a1c3c2d6 authored by Yori Fournier's avatar Yori Fournier
Browse files

run_pipeline now works, still need cleaning, tests and doc

parent 37ecb795
#!/bin/env bash
# load env
source pipe-env.sh
STEPS=(`./stoml ${PIPE_ROOT}/config.toml pipeline`)
for step in ${STEPS[@]};
do
if [ ! -d ${PIPE_ROOT}/${PIPELINE}/${step} ]
then
echo "ERROR: the step ${step} does not exists."
exit 1
fi
done
echo "" > current_pipeline.txt
for step_name in ${STEPS[@]};
do
echo
echo
echo "Start step: ${step_name}"
COMMIT=`./stoml ${PIPE_ROOT}/config.toml steps.${step_name}.commit`
DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${step_name}.dep`)
echo ">>> ${COMMIT}"
echo ">>> ${DEPS[@]}"
DEPS_DIR=()
for dep in ${DEPS[@]};
do
DEP=(`cat ${PIPE_ROOT}/current_pipeline.txt | grep ${dep}`)
DEPS_DIR=("${DEPS_DIR[@]}" "${DEP[1]}")
done
echo
echo ${DEPS_DIR[@]}
run_step.sh ${step_name} ${COMMIT} ${DEPS_DIR[@]}
done
exit 0
......@@ -98,6 +98,7 @@ then
STEP_DIR_NAME=($(ls -d ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}*))
echo "the Step was run already: ${STEP_DIR_NAME[0]}"
echo "${STEP_NAME} $(basename ${STEP_DIR_NAME[0]})" >> ${PIPE_ROOT}/current_pipeline.txt
if [[ ${#STEP_DIR_NAME[@]} > 1 ]];
then
......@@ -187,6 +188,7 @@ STEP_DIR_NAME=${STEP_NAME}-${short_deps_hash}-${short_step_hash}-${short_art_has
mv ${TMP_DIR} ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}
echo "${STEP_DIR_NAME}"
echo "${STEP_NAME} ${STEP_DIR_NAME}" >> ${PIPE_ROOT}/current_pipeline.txt
read -p "Do you want to remove the Job? (y/n)" resp
if [[ "${resp}" == "y" ]]; then
......
project_name = "ml-solar-plates"
pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-features", "extract-features-properties"]
pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-features"] # , "extract-features-properties"]
# RESOURCES: define STORAGE and CPU usage (need to cope with Quotas)
......@@ -23,7 +23,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
commit = "1ccb3969"
commit = "1a2e07805c3230789d662f73efc73c3a8e4bb3a3"
# the archives to be mounted
archives = ["ml_solar_plates_mock"]
......@@ -42,7 +42,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
commit = "0f33b92"
commit = "0f33b92bb17b6fc848585095c72b2beaec13c871"
# no archives to be mounted
archives = []
......
......@@ -2,42 +2,45 @@ apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-artifacts
namespace: default
spec:
accessModes:
- ReadWriteOnce
storageClassName: ""
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-artifacts-pv-claim
namespace: default
hostPath:
path: /mnt/ml_plates_artifacts
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-archives
namespace: default
spec:
accessModes:
- ReadWriteOnce
storageClassName: ""
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-archives-pv-claim
namespace: default
hostPath:
path: /mnt/ml_plates_archives
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ml-solar-plates-artifacts-pv-claim
namespace: default
spec:
storageClassName: standard
accessModes:
- ReadWriteOnce
storageClassName: ""
resources:
requests:
storage: 1Gi
......@@ -46,10 +49,11 @@ apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ml-solar-plates-archives-pv-claim
namespace: default
spec:
storageClassName: standard
accessModes:
- ReadWriteOnce
storageClassName: ""
resources:
requests:
storage: 1Gi
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment