-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile
133 lines (114 loc) 路 4.17 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
.DEFAULT_GOAL := help
###########################
# HELP
###########################
include *.mk
###########################
# VARIABLES
###########################
PROJECTNAME := selfclean
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD | tr / _)
PROJECT_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/)
COMMA := ,
DASH := -
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
# check if `netstat` is installed
ifeq (, $(shell which netstat))
$(error "Netstat executable not found, install it with `apt-get install net-tools`")
endif
# Check if Jupyter Port is already use and define an alternative
ifeq ($(origin PORT), undefined)
PORT_USED = $(shell netstat -tl | grep -E '(tcp|tcp6)' | grep -Eo '8888' | tail -n 1)
# Will fail if both ports 9999 and 10000 are used, I am sorry for that
NEXT_TCP_PORT = $(shell netstat -tl | grep -E '(tcp|tcp6)' | grep -Eo '[0-9]{4}' | sort | tail -n 1 | xargs -I '{}' expr {} + 1)
ifeq ($(PORT_USED), 8888)
PORT = $(NEXT_TCP_PORT)
else
PORT = 8888
endif
endif
# docker
ifeq ($(origin CONTAINER_NAME), undefined)
CONTAINER_NAME := default
endif
ifeq ($(origin LOCAL_DATA_DIR), undefined)
LOCAL_DATA_DIR := $$PWD/data/
endif
ifeq ($(origin GPU_ID), undefined)
GPU_ID := all
GPU_NAME := $(GPU_ID)
else
GPU_NAME = $(subst $(COMMA),$(DASH),$(GPU_ID))
endif
ifeq ("$(GPU)", "false")
ifeq (, $(shell which nvidia-smi))
GPU_ARGS := --shm-size 64G
else
GPU_ARGS := --gpus '"device="' --shm-size 64G
endif
DOCKER_CONTAINER_NAME := --name $(PROJECTNAME)_$(CONTAINER_NAME)
else
GPU_ARGS := --gpus '"device=$(GPU_ID)"' --shm-size 64G --ipc=host
DOCKER_CONTAINER_NAME := --name $(PROJECTNAME)_gpu_$(GPU_NAME)_$(CONTAINER_NAME)
endif
# count elements in comma-seperated GPU list
count = $(words $1)$(if $2,$(call count,$(wordlist 2,$(words $1),$1),$2))
GPU_LIST := $(subst $(COMMA),$(SPACE),$(GPU_ID))
NUM_GPUS := $(call count,$(GPU_LIST))
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
NUM_CORES := $(shell nproc)
else ifeq ($(UNAME_S),Darwin)
NUM_CORES := $(shell sysctl -n hw.ncpu)
else
NUM_CORES := 1
endif
# optimal number of threads is #cores/#gpus
NUM_THREADS := $(shell expr $(NUM_CORES) / $(NUM_GPUS))
DOCKER_ARGS := -v $$PWD:/workspace/ -v $(LOCAL_DATA_DIR):/data/ -p $(PORT):8888 --rm
DOCKER_CMD := docker run $(DOCKER_ARGS) $(GPU_ARGS) $(DOCKER_CONTAINER_NAME) -it $(PROJECTNAME):$(GIT_BRANCH)
###########################
# PROJECT UTILS
###########################
.PHONY: install
install: ##@Utils install the dependencies for the project
@python3 -m pip install -r requirements.txt
@pre-commit install
.PHONY: clean
clean: ##@Utils clean the project
@black .
@find . -name '*.pyc' -delete
@find . -name '__pycache__' -type d | xargs rm -fr
@rm -f .DS_Store
@rm -f .coverage coverage.xml report.xml
@rm -f -R .pytest_cache
@rm -f -R .idea
@rm -f -R tmp/
@rm -f -R cov_html/
@rm -f -R build/
@rm -f -R dist/
@rm -f -R selfclean.egg-info/
###########################
# DOCKER
###########################
_build:
@echo "Build image $(GIT_BRANCH)..."
@docker build -f Dockerfile -t $(PROJECTNAME):$(GIT_BRANCH) .
run_bash: _build ##@Docker run an interactive bash inside the docker image (default: GPU=true)
@echo "Running bash with GPU being $(GPU) and GPU_ID $(GPU_ID)"
$(DOCKER_CMD) /bin/bash; \
start_jupyter: _build ##@Docker start a jupyter notebook inside the docker image
@echo "Starting jupyter notebook"
@-docker rm $(DOCKER_CONTAINER_NAME)
$(DOCKER_CMD) /bin/bash -c "jupyter notebook --allow-root --ip 0.0.0.0 --port 8888"
.DEFAULT_GOAL := help
###########################
# TESTS
###########################
.PHONY: test
test: _build ##@Test run all tests in the project
$(DOCKER_CMD) /bin/bash -c "python3 -m coverage run -m pytest tests --junitxml=report.xml; coverage report -i --include=src/* --omit="src/ssl_library/*"; coverage xml -i --include=src/* --omit="src/ssl_library/*";"
.PHONY: unittest
unittest: _build ##@Test run all unittests in the project
$(DOCKER_CMD) /bin/bash -c "python3 -m coverage run -m pytest tests --junitxml=report.xml --ignore=tests/integration_tests; coverage report -i --include=src/* --omit="src/ssl_library/*"; coverage xml -i --include=src/* --omit="src/ssl_library/*";"