#/**
# * Copyright 2015 The Apache Software Foundation
# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements.  See the NOTICE file
# * distributed with this work for additional information
# * regarding copyright ownership.  The ASF licenses this file
# * to you under the Apache License, Version 2.0 (the
# * "License"); you may not use this file except in compliance
# * with the License.  You may obtain a copy of the License at
# *
# *     http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */

###################User Config Varaibles #############################
# third-party library installation folder
HOME_DIR := /home/wangwei/local

# must config the cudnn folder if using cudnn
CUDNN_DIR := $(HOME_DIR)/cudnn

CUDA_DIR := /usr/local/cuda

# Lib folder for system and external libs. You may need to change it.
LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(CUDNN_DIR)/lib64 $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib
# Header folder for system and external libs. You may need to change it.
INCLUDE_DIRS := ./include $(HOME_DIR)/include $(CUDNN_DIR)/include $(CUDA_DIR)/include
# g++ location, should support c++11, tested with 4.8.1
CXX := g++
CUCXX := nvcc

######################Setting Varialbes#######################################
LIBRARIES := glog protobuf openblas zmq czmq zookeeper_mt

ifneq ($(CUDA_DIR),)
	LIBRARIES := $(LIBRARIES) cublas cudart curand cudnn
endif

LDFLAGS := $(foreach librarydir, $(LIBRARY_DIRS), -L$(librarydir))\
	$(foreach library, $(LIBRARIES), -l$(library))
# Folder to store compiled files
BUILD_DIR := .libs
MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
ZK_FLAGS :=-DTHREADED -fpermissive
CXXFLAGS := -O2 -msse3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
	$(MSHADOW_FLAGS) -DUSE_CUDNN $(ZK_FLAGS)\
	-funroll-loops $(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
CUCXXFLAGS := -DUSE_CUDNN $(MSHADOW_FLAGS) -std=c++11 $(CUDA_ARCH) \
	$(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))

#Add device compile option
ifeq ($(CUDA_DIR),)
	MSHADOW_FLAGS := $(MSHADOW_FLAGS) -DCPU_ONLY
	CXXFLAGS := $(CXXFLAGS) -DCPU_ONLY
else
	CXXFLAGS := $(CXXFLAGS) -DUSE_GPU
endif

# find user defined .proto file, and then compute the corresponding .h, .cc
# files, which cannot be found by shell find, because they haven't been
# generated currently
PROTOS := $(shell find src/proto/ -name "*.proto")
PROTO_SRCS :=$(PROTOS:.proto=.pb.cc)
PROTO_HDRS :=$(patsubst src%, include%, $(PROTOS:.proto=.pb.h))
PROTO_OBJS :=$(addprefix $(BUILD_DIR)/, $(PROTO_SRCS:.cc=.o))

# each singa src file will generate a .o file
SINGA_SRCS := $(shell find src/ \( -path "src/test" -o -path "src/main.cc" -o -path "src/utils/tool.cc" \) \
	-prune -o \( -name "*.cc" -type f \) -print )
SINGA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_SRCS:.cc=.o)) \
	$(PROTO_OBJS) )
-include $(SINGA_OBJS:%.o=%.P)

TEST_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cc")
TEST_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_SRCS:.cc=.o)))
-include $(TEST_OBJS:%.o=%.P)

TEST_CUDA_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cu")
TEST_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_CUDA_SRCS:.cu=.o)))
-include $(TEST_CUDA_OBJS:%.o=%.P)

SINGA_CUDA_SRCS := $(shell find src/ \( -path "src/test" \) -prune -o \( -name "*.cu" -type f \) -print )
SINGA_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_CUDA_SRCS:.cu=.o)))
-include $(SINGA_CUDA_OBJS:%.o=%.P)

GTEST_SRC := include/gtest/gtest-all.cc
GTEST_HDR := include/gtest/gtest.h
GTEST_LIB := $(BUILD_DIR)/libgtest.a

OBJS := $(sort $(SINGA_OBJS) $(TEST_OBJS) )
CUOBJS := $(sort $(SINGA_CUDA_OBJS) $(TEST_CUDA_OBJS) )

########################Compilation Section###################################
.PHONY: singa test

singa: $(PROTO_OBJS) $(SINGA_OBJS) $(SINGA_CUDA_OBJS)
	$(CXX) -shared -o $(BUILD_DIR)/libsinga.so $(SINGA_OBJS)
	$(CXX) $(SINGA_OBJS) $(SINGA_CUDA_OBJS) src/main.cc -o singa $(CXXFLAGS) $(LDFLAGS)
	@echo
	$(CXX) $(BUILD_DIR)/libsinga.so src/utils/tool.cc -o singatool $(CXXFLAGS) $(LDFLAGS) -Wl,-unresolved-symbols=ignore-in-shared-libs
	@echo

loader: proto $(LOADER_OBJS)
	$(CXX) $(LOADER_OBJS) -o $(BUILD_DIR)/loader $(CXXFLAGS) $(LDFLAGS)
	@echo

test:  proto $(GTEST_LIB) $(TEST_OBJS) $(TEST_CUDA_OBJS) $(SINGA_OBJS) $(SINGA_CUDA_OBJS)
	$(CXX) $(TEST_OBJS) $(TEST_CUDA_OBJS) include/gtest/gtest_main.cc $(GTEST_LIB) \
		$(SINGA_OBJS) $(SINGA_CUDA_OBJS) -o $(BUILD_DIR)/test $(CXXFLAGS) $(LDFLAGS)
	@echo

$(GTEST_LIB): $(GTEST_HDR) $(GTEST_SRC)
	$(CXX) $(GTEST_SRC) -c -o $(BUILD_DIR)/gtest-all.o $(CXXFLAGS)
	ar -rv $(GTEST_LIB) $(BUILD_DIR)/gtest-all.o

# compile all files
$(OBJS):$(BUILD_DIR)/%.o : %.cc
	@mkdir -p $(dir $@)
	$(CXX) $<  $(CXXFLAGS) -MMD -c -o $@
	cp $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.P; \
	sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
		-e '/^$$/ d' -e 's/$$/ :/' < $(BUILD_DIR)/$*.d >> $(BUILD_DIR)/$*.P; \
	rm -f $*.d

$(CUOBJS):$(BUILD_DIR)/%.o : %.cu
	@mkdir -p $(dir $@)
	$(CUCXX) $< -c -o $@ $(CUCXXFLAGS)
	cp $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.P; \
	sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
		-e '/^$$/ d' -e 's/$$/ :/' < $(BUILD_DIR)/$*.d >> $(BUILD_DIR)/$*.P; \
	rm -f $*.d

proto: $(PROTO_OBJS)

$(PROTO_SRCS): $(PROTOS)
	protoc --proto_path=src/proto --cpp_out=src/proto $(PROTOS)
	mkdir -p include/proto/
	cp src/proto/*.pb.h include/singa/proto/
	mkdir -p tool/pb2/
	touch tool/pb2/__init__.py
	protoc --proto_path=src/proto --python_out=tool/pb2/ $(PROTOS)
	@echo

clean:
	rm -rf *.a *.so
	rm -rf include/proto/*
	rm -rf src/proto/*.pb.h src/proto/*.pb.cc
	rm -rf tool/pb2/*
	rm -rf $(BUILD_DIR)
	@echo