| /************************************************************ |
| * |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| * |
| *************************************************************/ |
| |
| /** |
| * The code is adapted from Caffe under BSD 2 Clause license. |
| * All contributions by the University of California: |
| * Copyright (c) 2014, The Regents of the University of California (Regents) |
| * All rights reserved. |
| * All other contributions: |
| * Copyright (c) 2014, the respective contributors |
| * All rights reserved. |
| * Caffe uses a shared copyright model: each contributor holds copyright over |
| * their contributions to Caffe. The project versioning records all such |
| * contribution and copyright details. If a contributor wants to further mark |
| * their specific copyright on a particular contribution, they should indicate |
| * their copyright solely in the commit message of the change when it is |
| * committed. |
| */ |
| |
| #include "singa/utils/common.h" |
| |
| #include <sys/ioctl.h> |
| #include <sys/socket.h> |
| #include <sys/stat.h> |
| #include <sys/types.h> |
| |
| #include <netinet/in.h> |
| #include <net/if.h> |
| #include <arpa/inet.h> |
| |
| #include <stdarg.h> |
| #include <stdio.h> |
| #include <time.h> |
| #include <unistd.h> |
| #include <fcntl.h> |
| #include <cfloat> |
| |
| #include <fstream> |
| |
| #include <glog/logging.h> |
| #include <google/protobuf/io/coded_stream.h> |
| #include <google/protobuf/io/zero_copy_stream_impl.h> |
| #include <google/protobuf/text_format.h> |
| |
| namespace singa { |
| |
| const int kBufLen = 1024; |
| |
| string IntVecToString(const vector<int>& vec) { |
| string disp = "("; |
| for (int x : vec) |
| disp += std::to_string(x) + ", "; |
| return disp + ")"; |
| } |
| |
| /** |
| * * Formatted string. |
| * */ |
| string VStringPrintf(string fmt, va_list l) { |
| char buffer[4096]; |
| vsnprintf(buffer, sizeof(buffer), fmt.c_str(), l); |
| return string(buffer); |
| } |
| |
| /** |
| * * Formatted string. |
| * */ |
| string StringPrintf(string fmt, ...) { |
| va_list l; |
| va_start(l, fmt); // fmt.AsString().c_str()); |
| string result = VStringPrintf(fmt, l); |
| va_end(l); |
| return result; |
| } |
| |
| int ArgPos(int argc, char** arglist, const char* arg) { |
| for (int i = 0; i < argc; i++) { |
| if (strcmp(arglist[i], arg) == 0) { |
| return i; |
| } |
| } |
| return -1; |
| } |
| |
| void CreateFolder(const string name) { |
| struct stat buffer; |
| if (stat(name.c_str(), &buffer) != 0) { |
| mkdir(name.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); |
| CHECK_EQ(stat(name.c_str(), &buffer), 0); |
| } |
| } |
| |
| const vector<vector<int>> Slice(int num, const vector<int>& sizes) { |
| vector<vector<int>> slices; |
| if (num == 0) |
| return slices; |
| int avg = 0; |
| for (int x : sizes) |
| avg += x; |
| avg = avg / num + avg % num; |
| int diff = avg / 10; |
| // DLOG(INFO) << "Slicer, param avg = " << avg << ", diff = " << diff; |
| |
| int capacity = avg, nbox = 0; |
| for (int x : sizes) { |
| vector<int> slice; |
| string slicestr = ""; |
| while (x > 0) { |
| int size = 0; |
| if (capacity >= x) { |
| capacity -= x; |
| size = x; |
| x = 0; |
| } else if (capacity + diff >= x) { |
| size = x; |
| x = 0; |
| capacity = 0; |
| } else if (capacity >= diff) { |
| x -= capacity; |
| size = capacity; |
| capacity = avg; |
| nbox++; |
| } else { |
| capacity = avg; |
| nbox++; |
| } |
| if (size) { |
| slice.push_back(size); |
| slicestr += ", " + std::to_string(size); |
| } |
| } |
| // DLOG(INFO) << slicestr; |
| slices.push_back(slice); |
| } |
| CHECK_LE(nbox, num); |
| return slices; |
| } |
| |
| const vector<int> PartitionSlices(int num, const vector<int>& slices) { |
| vector<int> slice2box; |
| if (num == 0) |
| return slice2box; |
| int avg = 0; |
| for (int x : slices) |
| avg += x; |
| avg = avg / num + avg % num; |
| int box = avg, boxid = 0, diff = avg / 10; |
| for (auto it = slices.begin(); it != slices.end();) { |
| int x = *it; |
| if (box >= x) { |
| box -= x; |
| slice2box.push_back(boxid); |
| it++; |
| } else if (box + diff >= x) { |
| slice2box.push_back(boxid); |
| it++; |
| box = 0; |
| } else { |
| box = avg; |
| boxid++; |
| } |
| } |
| CHECK_EQ(slice2box.size(), slices.size()); |
| int previd = -1; |
| string disp; |
| for (size_t i = 0; i < slice2box.size(); i++) { |
| if (previd != slice2box[i]) { |
| previd = slice2box[i]; |
| disp += " box = " +std::to_string(previd) + ":"; |
| } |
| disp += " " + std::to_string(slices[i]); |
| } |
| return slice2box; |
| } |
| |
| int gcd(int a, int b) { |
| for (;;) { |
| if (a == 0) return b; |
| b %= a; |
| if (b == 0) return a; |
| a %= b; |
| } |
| } |
| |
| int LeastCommonMultiple(int a, int b) { |
| int temp = gcd(a, b); |
| return temp ? (a / temp * b) : 0; |
| } |
| |
| string GetHostIP() { |
| int fd; |
| struct ifreq ifr; |
| fd = socket(AF_INET, SOCK_DGRAM, 0); |
| /* I want to get an IPv4 IP address */ |
| ifr.ifr_addr.sa_family = AF_INET; |
| /* I want IP address attached to "eth0" */ |
| strncpy(ifr.ifr_name, "eth0", IFNAMSIZ-1); |
| ioctl(fd, SIOCGIFADDR, &ifr); |
| close(fd); |
| string ip(inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr)); |
| /* display result */ |
| LOG(INFO) << "Host IP = " << ip; |
| return ip; |
| } |
| |
| void SetupLog(const string& log_dir, const string& model) { |
| // TODO(wangwei) check if NFS, then create folder using script, otherwise |
| // may have problems due to multiple processes create the same folder. |
| CreateFolder(log_dir); |
| string warn = log_dir + "/" + model + "-warn-"; |
| string info = log_dir + "/" + model + "-info-"; |
| string error = log_dir + "/" + model + "-error-"; |
| string fatal = log_dir + "/" + model + "-fatal-"; |
| google::SetLogDestination(google::WARNING, warn.c_str()); |
| google::SetLogDestination(google::INFO, info.c_str()); |
| google::SetLogDestination(google::ERROR, error.c_str()); |
| google::SetLogDestination(google::FATAL, fatal.c_str()); |
| } |
| |
| Metric::Metric(const string& str) { |
| ParseFrom(str); |
| } |
| |
| void Metric::Add(const string& name, float value) { |
| Add(name, value, 1); |
| } |
| void Metric::Add(const string& name, float value, int count) { |
| if (entry_.find(name) == entry_.end()) { |
| entry_[name] = std::make_pair(1, value); |
| } else { |
| auto& e = entry_.at(name); |
| e.first += count; |
| e.second += value; |
| } |
| } |
| |
| void Metric::Reset() { |
| for (auto& e : entry_) { |
| e.second.first = 0; |
| e.second.second = 0; |
| } |
| } |
| |
| string Metric::ToLogString() const { |
| string ret; |
| size_t k = 0; |
| for (auto e : entry_) { |
| ret += e.first + " = "; |
| ret += std::to_string(e.second.second / e.second.first); |
| if (++k < entry_.size()) |
| ret += ", "; |
| } |
| return ret; |
| } |
| |
| string Metric::ToString() const { |
| MetricProto proto; |
| for (auto e : entry_) { |
| proto.add_name(e.first); |
| proto.add_count(e.second.first); |
| proto.add_val(e.second.second); |
| } |
| string ret; |
| proto.SerializeToString(&ret); |
| return ret; |
| } |
| |
| void Metric::ParseFrom(const string& msg) { |
| MetricProto proto; |
| proto.ParseFromString(msg); |
| Reset(); |
| for (int i = 0; i < proto.name_size(); i++) { |
| entry_[proto.name(i)] = std::make_pair(proto.count(i), proto.val(i)); |
| } |
| } |
| |
| |
| /*************Below functions are adapted from Caffe ************/ |
| using google::protobuf::io::CodedInputStream; |
| using google::protobuf::io::FileInputStream; |
| using google::protobuf::io::FileOutputStream; |
| using google::protobuf::io::ZeroCopyInputStream; |
| |
| |
| void Im2col(const float* data_im, const int channels, |
| const int height, const int width, const int kernel_h, const int kernel_w, |
| const int pad_h, const int pad_w, const int stride_h, const int stride_w, |
| float* data_col) { |
| int height_col = (height + 2 * pad_h - kernel_h) / stride_h + 1; |
| int width_col = (width + 2 * pad_w - kernel_w) / stride_w + 1; |
| int channels_col = channels * kernel_h * kernel_w; |
| for (int c = 0; c < channels_col; ++c) { |
| int w_offset = c % kernel_w; |
| int h_offset = (c / kernel_w) % kernel_h; |
| int c_im = c / kernel_h / kernel_w; |
| for (int h = 0; h < height_col; ++h) { |
| for (int w = 0; w < width_col; ++w) { |
| int h_pad = h * stride_h - pad_h + h_offset; |
| int w_pad = w * stride_w - pad_w + w_offset; |
| if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) |
| data_col[(c * height_col + h) * width_col + w] = |
| data_im[(c_im * height + h_pad) * width + w_pad]; |
| else |
| data_col[(c * height_col + h) * width_col + w] = 0; |
| } |
| } |
| } |
| } |
| |
| void Col2im(const float* data_col, const int channels, |
| const int height, const int width, const int patch_h, const int patch_w, |
| const int pad_h, const int pad_w, const int stride_h, const int stride_w, |
| float* data_im) { |
| memset(data_im, 0, height * width * channels * sizeof(float)); |
| int height_col = (height + 2 * pad_h - patch_h) / stride_h + 1; |
| int width_col = (width + 2 * pad_w - patch_w) / stride_w + 1; |
| int channels_col = channels * patch_h * patch_w; |
| for (int c = 0; c < channels_col; ++c) { |
| int w_offset = c % patch_w; |
| int h_offset = (c / patch_w) % patch_h; |
| int c_im = c / patch_h / patch_w; |
| for (int h = 0; h < height_col; ++h) { |
| for (int w = 0; w < width_col; ++w) { |
| int h_pad = h * stride_h - pad_h + h_offset; |
| int w_pad = w * stride_w - pad_w + w_offset; |
| if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) |
| data_im[(c_im * height + h_pad) * width + w_pad] += |
| data_col[(c * height_col + h) * width_col + w]; |
| } |
| } |
| } |
| } |
| |
| void ForwardMaxPooling(const float* bottom, const int num, const int channels, |
| const int height, const int width, const int kernel_h, const int kernel_w, |
| const int pad_h, const int pad_w, const int stride_h, const int stride_w, |
| float* top, float* mask) { |
| int top_height = (height + pad_h * 2 -kernel_h) / stride_h + 1; |
| int top_width = (width + pad_w * 2 -kernel_w) / stride_w + 1; |
| int top_count = num * top_height * top_width * channels; |
| for (int i = 0; i < top_count; i++) { |
| mask[i] = -1; |
| top[i] = -FLT_MAX; |
| } |
| const int bottom_offset = height * width; |
| const int top_offset = top_height * top_width; |
| // The main loop |
| for (int n = 0; n < num; ++n) { |
| for (int c = 0; c < channels; ++c) { |
| for (int ph = 0; ph < top_height; ++ph) { |
| for (int pw = 0; pw < top_width; ++pw) { |
| int hstart = ph * stride_h - pad_h; |
| int wstart = pw * stride_w - pad_w; |
| int hend = std::min(hstart + kernel_h, height); |
| int wend = std::min(wstart + kernel_w, width); |
| hstart = std::max(hstart, 0); |
| wstart = std::max(wstart, 0); |
| const int top_index = ph * top_width + pw; |
| for (int h = hstart; h < hend; ++h) { |
| for (int w = wstart; w < wend; ++w) { |
| const int index = h * width + w; |
| if (bottom[index] > top[top_index]) { |
| top[top_index] = bottom[index]; |
| mask[top_index] = index; |
| } |
| } |
| } |
| } |
| } |
| // compute offset |
| bottom += bottom_offset; |
| top += top_offset; |
| mask += top_offset; |
| } |
| } |
| } |
| |
| void BackwardMaxPooling(const float* top, const float* mask, const int num, |
| const int channels, const int height, const int width, |
| const int kernel_h, const int kernel_w, const int pad_h, const int pad_w, |
| const int stride_h, const int stride_w, |
| float* bottom) { |
| int top_height = (height + pad_h * 2 -kernel_h) / stride_h + 1; |
| int top_width = (width + pad_w * 2 -kernel_w) / stride_w + 1; |
| const int top_offset = top_height * top_width; |
| const int bottom_offset = height * width; |
| memset(bottom, 0, sizeof(float) * num * channels * bottom_offset); |
| for (int n = 0; n < num; ++n) { |
| for (int c = 0; c < channels; ++c) { |
| for (int ph = 0; ph < top_height; ++ph) { |
| for (int pw = 0; pw < top_width; ++pw) { |
| const int top_idx = ph * top_width + pw; |
| const int bottom_idx = static_cast<int>(mask[top_idx]); |
| bottom[bottom_idx] += top[top_idx]; |
| } |
| } |
| top += top_offset; |
| mask += top_offset; |
| bottom += bottom_offset; |
| } |
| } |
| } |
| |
| void ForwardAvgPooling(const float* bottom, const int num, const int channels, |
| const int height, const int width, const int kernel_h, const int kernel_w, |
| const int pad_h, const int pad_w, const int stride_h, const int stride_w, |
| float* top) { |
| int top_height = (height + pad_h * 2 -kernel_h) / stride_h + 1; |
| int top_width = (width + pad_w * 2 -kernel_w) / stride_w + 1; |
| int top_count = num * top_height * top_width * channels; |
| for (int i = 0; i < top_count; i++) { |
| top[i] = 0; |
| } |
| const int bottom_offset = height * width; |
| const int top_offset = top_height * top_width; |
| // The main loop |
| for (int n = 0; n < num; ++n) { |
| for (int c = 0; c < channels; ++c) { |
| for (int ph = 0; ph < top_height; ++ph) { |
| for (int pw = 0; pw < top_width; ++pw) { |
| int hstart = ph * stride_h - pad_h; |
| int wstart = pw * stride_w - pad_w; |
| int hend = std::min(hstart + kernel_h, height+pad_h); |
| int wend = std::min(wstart + kernel_w, width+pad_w); |
| int pool_size = (hend-hstart) * (wend-wstart); |
| hstart = std::max(hstart, 0); |
| wstart = std::max(wstart, 0); |
| hend = std::min(hend, height); |
| wend = std::min(wend, width); |
| const int top_index = ph * top_width + pw; |
| for (int h = hstart; h < hend; ++h) { |
| for (int w = wstart; w < wend; ++w) { |
| const int index = h * width + w; |
| top[top_index] += bottom[index]; |
| } |
| } |
| top[top_index] /= pool_size; |
| } |
| } |
| // compute offset |
| bottom += bottom_offset; |
| top += top_offset; |
| } |
| } |
| } |
| |
| void BackwardAvgPooling(const float* top, const int num, const int channels, |
| const int height, const int width, const int kernel_h, const int kernel_w, |
| const int pad_h, const int pad_w, const int stride_h, const int stride_w, |
| float* bottom) { |
| int top_height = (height + pad_h * 2 -kernel_h) / stride_h + 1; |
| int top_width = (width + pad_w * 2 -kernel_w) / stride_w + 1; |
| const int top_offset = top_height * top_width; |
| const int bottom_offset = height * width; |
| memset(bottom, 0, sizeof(float) * num * channels * bottom_offset); |
| for (int n = 0; n < num; ++n) { |
| for (int c = 0; c < channels; ++c) { |
| for (int ph = 0; ph < top_height; ++ph) { |
| for (int pw = 0; pw < top_width; ++pw) { |
| int hstart = ph * stride_h - pad_h; |
| int wstart = pw * stride_w - pad_w; |
| int hend = std::min(hstart + kernel_h, height+pad_h); |
| int wend = std::min(wstart + kernel_w, width+pad_w); |
| int pool_size = (hend-hstart) * (wend-wstart); |
| hstart = std::max(hstart, 0); |
| wstart = std::max(wstart, 0); |
| hend = std::min(hend, height); |
| wend = std::min(wend, width); |
| const int top_index = ph * top_width + pw; |
| for (int h = hstart; h < hend; ++h) { |
| for (int w = wstart; w < wend; ++w) { |
| const int index = h * width + w; |
| bottom[index] += top[top_index] / pool_size; |
| } |
| } |
| } |
| } |
| top += top_offset; |
| bottom += bottom_offset; |
| } |
| } |
| } |
| |
| void ReadProtoFromTextFile(const char* filename, Message* proto) { |
| int fd = open(filename, O_RDONLY); |
| CHECK_NE(fd, -1) << "File not found: " << filename; |
| FileInputStream* input = new FileInputStream(fd); |
| CHECK(google::protobuf::TextFormat::Parse(input, proto)); |
| delete input; |
| close(fd); |
| } |
| |
| void WriteProtoToTextFile(const Message& proto, const char* filename) { |
| int fd = open(filename, O_WRONLY | O_CREAT, 0644); |
| FileOutputStream* output = new FileOutputStream(fd); |
| CHECK(google::protobuf::TextFormat::Print(proto, output)); |
| delete output; |
| close(fd); |
| } |
| |
| void ReadProtoFromBinaryFile(const char* filename, Message* proto) { |
| int fd = open(filename, O_RDONLY); |
| CHECK_NE(fd, -1) << "File not found: " << filename; |
| ZeroCopyInputStream* raw_input = new FileInputStream(fd); |
| CodedInputStream* coded_input = new CodedInputStream(raw_input); |
| // upper limit 512MB, warning threshold 256MB |
| coded_input->SetTotalBytesLimit(536870912, 268435456); |
| CHECK(proto->ParseFromCodedStream(coded_input)); |
| delete coded_input; |
| delete raw_input; |
| close(fd); |
| } |
| |
| void WriteProtoToBinaryFile(const Message& proto, const char* filename) { |
| int fd = open(filename, O_CREAT|O_WRONLY|O_TRUNC, 0644); |
| CHECK_NE(fd, -1) << "File cannot open: " << filename; |
| CHECK(proto.SerializeToFileDescriptor(fd)); |
| } |
| |
| |
| |
| void WriteStringToTextFile(const string& filename, const string& context) { |
| std::ofstream ofs; |
| ofs.open(filename); |
| CHECK(ofs.is_open()) << "Can't write to file: " << filename; |
| ofs << context; |
| ofs.flush(); |
| ofs.close(); |
| } |
| |
| |
| const vector<std::pair<string, float>> GetMetricFromString(const string& disp) { |
| size_t pos = 0; |
| vector<string> terms; |
| while (pos != string::npos) { |
| auto next = disp.find_first_of(" ,", pos); // delimiter: space or comma |
| if (next != string::npos) { |
| terms.push_back(disp.substr(pos, next - pos)); |
| pos = disp.find_first_not_of(" ,", next + 1); |
| } else { |
| break; |
| } |
| } |
| if (pos != string::npos) |
| terms.push_back(disp.substr(pos)); |
| vector<std::pair<string, float>> ret; |
| for (unsigned i = 0; i < terms.size(); i++) { |
| if (terms[i] == "=") { |
| CHECK_GE(i, 1); |
| CHECK_LT(i, terms.size() - 1) << "terms[i] = " << terms[i]; |
| ret.push_back(std::make_pair(terms[i-1], std::stof(terms[i + 1]))); |
| } |
| } |
| return ret; |
| } |
| } // namespace singa |