| /* |
| * Copyright 2010 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // Author: jmarantz@google.com (Joshua Marantz) |
| |
| #include "pagespeed/kernel/base/stdio_file_system.h" |
| |
| #include <errno.h> |
| #include <sys/stat.h> |
| #ifdef WIN32 |
| #include <direct.h> |
| #include <io.h> |
| #include <stdio.h> |
| #include <string.h> |
| #include <windows.h> |
| #else |
| #include <dirent.h> |
| #include <unistd.h> |
| #endif // WIN32 |
| |
| #include <algorithm> |
| #include <cstddef> |
| #include <cstdio> |
| #include <cstdlib> |
| #include <limits> |
| |
| #include "base/logging.h" |
| #include "pagespeed/kernel/base/timer.h" |
| #include "pagespeed/kernel/base/basictypes.h" |
| #include "pagespeed/kernel/base/debug.h" |
| #include "pagespeed/kernel/base/file_system.h" |
| #include "pagespeed/kernel/base/message_handler.h" |
| #include "pagespeed/kernel/base/null_message_handler.h" |
| #include "pagespeed/kernel/base/string.h" |
| #include "pagespeed/kernel/base/string_util.h" |
| |
| namespace { |
| // The st_blocks field returned by stat is the number of 512B blocks allocated |
| // for the files. (While POSIX doesn't specify this, it's the proper value on |
| // at least Linux, FreeBSD, and OS X). |
| const int kBlockSize = 512; |
| |
| static const char kOutstandingOps[] = "stdio_fs_outstanding_ops"; |
| static const char kSlowOps[] = "stdio_fs_slow_ops"; |
| static const char kTotalOps[] = "stdio_fs_total_ops"; |
| |
| } // namespace |
| |
| namespace net_instaweb { |
| |
| // Helper class to factor out common implementation details between Input and |
| // Output files, in lieu of multiple inheritance. |
| class StdioFileHelper { |
| public: |
| StdioFileHelper(FILE* f, const StringPiece& filename, StdioFileSystem* fs) |
| : file_(f), |
| file_system_(fs), |
| start_us_(0) { |
| filename.CopyToString(&filename_); |
| } |
| |
| ~StdioFileHelper() { |
| CHECK(file_ == NULL); |
| } |
| |
| void ReportError(MessageHandler* message_handler, const char* context) { |
| message_handler->Message(kError, "%s: %s %d(%s)", filename_.c_str(), |
| context, errno, strerror(errno)); |
| } |
| |
| bool Close(MessageHandler* message_handler) { |
| bool ret = true; |
| if (file_ != stdout && file_ != stderr && file_ != stdin) { |
| if (fclose(file_) != 0) { |
| ReportError(message_handler, "closing file"); |
| ret = false; |
| } |
| } |
| file_ = NULL; |
| return ret; |
| } |
| |
| void StartTimer() { |
| start_us_ = file_system_->StartTimer(); |
| } |
| |
| void EndTimer(const char* operation) { |
| file_system_->EndTimer(filename_.c_str(), operation, start_us_); |
| } |
| |
| |
| FILE* file_; |
| GoogleString filename_; |
| StdioFileSystem* file_system_; |
| int64 start_us_; |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(StdioFileHelper); |
| }; |
| |
| class StdioInputFile : public FileSystem::InputFile { |
| public: |
| StdioInputFile(FILE* f, const StringPiece& filename, StdioFileSystem* fs) |
| : file_helper_(f, filename, fs) { |
| } |
| |
| virtual bool ReadFile(GoogleString* buf, MessageHandler* message_handler) { |
| bool ret = false; |
| struct stat statbuf; |
| file_helper_.StartTimer(); |
| if ((fstat(fileno(file_helper_.file_), &statbuf) < 0)) { |
| file_helper_.ReportError(message_handler, "stating file"); |
| } else { |
| buf->resize(statbuf.st_size); |
| int nread = fread(&(*buf)[0], 1, statbuf.st_size, file_helper_.file_); |
| if (nread != statbuf.st_size) { |
| file_helper_.ReportError(message_handler, "reading file"); |
| } else { |
| ret = true; |
| } |
| } |
| file_helper_.EndTimer("ReadFile"); |
| return ret; |
| } |
| |
| virtual int Read(char* buf, int size, MessageHandler* message_handler) { |
| file_helper_.StartTimer(); |
| int ret = fread(buf, 1, size, file_helper_.file_); |
| if ((ret == 0) && (ferror(file_helper_.file_) != 0)) { |
| file_helper_.ReportError(message_handler, "reading file"); |
| } |
| file_helper_.EndTimer("read"); |
| return ret; |
| } |
| |
| virtual bool Close(MessageHandler* message_handler) { |
| return file_helper_.Close(message_handler); |
| } |
| |
| virtual const char* filename() { return file_helper_.filename_.c_str(); } |
| |
| private: |
| StdioFileHelper file_helper_; |
| |
| DISALLOW_COPY_AND_ASSIGN(StdioInputFile); |
| }; |
| |
| class StdioOutputFile : public FileSystem::OutputFile { |
| public: |
| StdioOutputFile(FILE* f, const StringPiece& filename, StdioFileSystem* fs) |
| : file_helper_(f, filename, fs) { |
| } |
| |
| virtual bool Write(const StringPiece& buf, MessageHandler* handler) { |
| file_helper_.StartTimer(); |
| size_t bytes_written = |
| fwrite(buf.data(), 1, buf.size(), file_helper_.file_); |
| bool ret = (bytes_written == buf.size()); |
| if (!ret) { |
| file_helper_.ReportError(handler, "writing file"); |
| } |
| file_helper_.EndTimer("write"); |
| return ret; |
| } |
| |
| virtual bool Flush(MessageHandler* message_handler) { |
| bool ret = true; |
| if (fflush(file_helper_.file_) != 0) { |
| file_helper_.ReportError(message_handler, "flushing file"); |
| ret = false; |
| } |
| return ret; |
| } |
| |
| virtual bool Close(MessageHandler* message_handler) { |
| return file_helper_.Close(message_handler); |
| } |
| |
| virtual const char* filename() { return file_helper_.filename_.c_str(); } |
| |
| virtual bool SetWorldReadable(MessageHandler* message_handler) { |
| bool ret = true; |
| #ifdef WIN32 |
| const char* filename = file_helper_.filename_.c_str(); |
| ret = (_chmod(filename, _S_IREAD) == 0); |
| #else |
| int fd = fileno(file_helper_.file_); |
| ret = (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0); |
| #endif // WIN32 |
| if (!ret) { |
| file_helper_.ReportError(message_handler, "setting world-readable"); |
| } |
| return ret; |
| } |
| |
| private: |
| StdioFileHelper file_helper_; |
| |
| DISALLOW_COPY_AND_ASSIGN(StdioOutputFile); |
| }; |
| |
| StdioFileSystem::StdioFileSystem() |
| : slow_file_latency_threshold_us_(0), |
| timer_(NULL), |
| statistics_(NULL), |
| outstanding_ops_(NULL), |
| slow_ops_(NULL), |
| total_ops_(NULL) { |
| } |
| |
| StdioFileSystem::~StdioFileSystem() { |
| } |
| |
| void StdioFileSystem::InitStats(Statistics* stats) { |
| stats->AddUpDownCounter(kOutstandingOps); |
| stats->AddVariable(kSlowOps); |
| stats->AddVariable(kTotalOps); |
| } |
| |
| void StdioFileSystem::TrackTiming(int64 slow_file_latency_threshold_us, |
| Timer* timer, Statistics* stats, |
| MessageHandler* handler) { |
| slow_file_latency_threshold_us_ = slow_file_latency_threshold_us; |
| timer_ = timer; |
| statistics_ = stats; |
| outstanding_ops_ = stats->GetUpDownCounter(kOutstandingOps); |
| slow_ops_ = stats->GetVariable(kSlowOps); |
| total_ops_ = stats->GetVariable(kTotalOps); |
| message_handler_ = handler; |
| } |
| |
| int64 StdioFileSystem::StartTimer() { |
| if (timer_ == NULL) { |
| return 0; |
| } |
| if (outstanding_ops_ != NULL) { |
| outstanding_ops_->Add(1); |
| } |
| if (total_ops_ != NULL) { |
| total_ops_->Add(1); |
| } |
| return timer_->NowUs(); |
| } |
| |
| void StdioFileSystem::EndTimer(const char* filename, const char* operation, |
| int64 start_us) { |
| if (outstanding_ops_ != NULL) { |
| outstanding_ops_->Add(-1); |
| } |
| if (timer_ != NULL) { |
| int64 end_us = timer_->NowUs(); |
| int64 latency_us = end_us - start_us; |
| if (latency_us > slow_file_latency_threshold_us_) { |
| if (slow_ops_ != NULL) { |
| slow_ops_->Add(1); |
| } |
| message_handler_->Message( |
| kError, "Slow %s operation on file %s: %gms; " |
| "configure SlowFileLatencyUs to change threshold\n", |
| operation, filename, latency_us / 1000.0); |
| } |
| } |
| } |
| |
| int StdioFileSystem::MaxPathLength(const StringPiece& base) const { |
| #ifdef WIN32 |
| return MAX_PATH; |
| #else |
| const int kMaxInt = std::numeric_limits<int>::max(); |
| |
| long limit = pathconf(base.as_string().c_str(), _PC_PATH_MAX); // NOLINT |
| if (limit < 0) { |
| // pathconf failed. |
| return FileSystem::MaxPathLength(base); |
| } else if (limit > kMaxInt) { |
| // As pathconf returns a long, we may have to clamp it. |
| return kMaxInt; |
| } else { |
| return limit; |
| } |
| #endif // WIN32 |
| } |
| |
| FileSystem::InputFile* StdioFileSystem::OpenInputFile( |
| const char* filename, MessageHandler* message_handler) { |
| FileSystem::InputFile* input_file = NULL; |
| FILE* f = fopen(filename, "r"); |
| if (f == NULL) { |
| message_handler->Error(filename, 0, "opening input file: %s", |
| strerror(errno)); |
| } else { |
| input_file = new StdioInputFile(f, filename, this); |
| } |
| return input_file; |
| } |
| |
| |
| FileSystem::OutputFile* StdioFileSystem::OpenOutputFileHelper( |
| const char* filename, bool append, MessageHandler* message_handler) { |
| FileSystem::OutputFile* output_file = NULL; |
| if (strcmp(filename, "-") == 0) { |
| output_file = new StdioOutputFile(stdout, "<stdout>", this); |
| } else { |
| const char* mode = append ? "a" : "w"; |
| FILE* f = fopen(filename, mode); |
| if (f == NULL) { |
| message_handler->Error(filename, 0, |
| "opening output file: %s", strerror(errno)); |
| } else { |
| output_file = new StdioOutputFile(f, filename, this); |
| } |
| } |
| return output_file; |
| } |
| |
| FileSystem::OutputFile* StdioFileSystem::OpenTempFileHelper( |
| const StringPiece& prefix, MessageHandler* message_handler) { |
| // TODO(jmarantz): As jmaessen points out, mkstemp warns "Don't use |
| // this function, use tmpfile(3) instead. It is better defined and |
| // more portable." However, tmpfile does not allow a location to be |
| // specified. I'm not 100% sure if that's going to be work well for |
| // us. More importantly, our usage scenario is that we will be |
| // closing the file and renaming it to a permanent name. tmpfiles |
| // automatically are deleted when they are closed. |
| int prefix_len = prefix.length(); |
| static char mkstemp_hook[] = "XXXXXX"; |
| char* template_name = new char[prefix_len + sizeof(mkstemp_hook)]; |
| memcpy(template_name, prefix.data(), prefix_len); |
| memcpy(template_name + prefix_len, mkstemp_hook, sizeof(mkstemp_hook)); |
| #ifdef WIN32 |
| int fd = _mktemp_s(template_name, prefix_len + sizeof(mkstemp_hook)); |
| #else |
| int fd = mkstemp(template_name); |
| #endif // WIN32 |
| OutputFile* output_file = NULL; |
| if (fd < 0) { |
| message_handler->Error(template_name, 0, |
| "opening temp file: %s", strerror(errno)); |
| } else { |
| #ifdef WIN32 |
| FILE* f = _fdopen(fd, "w"); |
| if (f == NULL) { |
| _close(fd); |
| #else |
| FILE* f = fdopen(fd, "w"); |
| if (f == NULL) { |
| close(fd); |
| #endif |
| |
| // If we failed to open the temp file, silently clean it before returning. |
| message_handler->Error(template_name, 0, |
| "re-opening temp file: %s", strerror(errno)); |
| NullMessageHandler null_message_handler; |
| RemoveFile(template_name, &null_message_handler); |
| } else { |
| output_file = new StdioOutputFile(f, template_name, this); |
| } |
| } |
| |
| delete [] template_name; |
| return output_file; |
| } |
| |
| |
| bool StdioFileSystem::RemoveFile(const char* filename, |
| MessageHandler* handler) { |
| bool ret = (remove(filename) == 0); |
| if (!ret) { |
| handler->Message(kError, "Failed to delete file %s: %s", |
| filename, strerror(errno)); |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::RenameFileHelper(const char* old_file, |
| const char* new_file, |
| MessageHandler* handler) { |
| bool ret = (rename(old_file, new_file) == 0); |
| if (!ret) { |
| handler->Message(kError, "Failed to rename file %s to %s: %s", |
| old_file, new_file, strerror(errno)); |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::MakeDir(const char* path, MessageHandler* handler) { |
| #ifdef WIN32 |
| bool ret = (_mkdir(path) == 0); |
| #else |
| // Mode 0777 makes the file use standard umask permissions. |
| bool ret = (mkdir(path, 0777) == 0); |
| #endif // WIN32 |
| if (!ret) { |
| handler->Message(kError, "Failed to make directory %s: %s", |
| path, strerror(errno)); |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::RemoveDir(const char* path, MessageHandler* handler) { |
| #ifdef WIN32 |
| bool ret = (_rmdir(path) == 0); |
| #else |
| bool ret = (rmdir(path) == 0); |
| #endif // WIN32 |
| if (!ret) { |
| handler->Message(kError, "Failed to remove directory %s: %s", |
| path, strerror(errno)); |
| } |
| return ret; |
| } |
| |
| BoolOrError StdioFileSystem::Exists(const char* path, MessageHandler* handler) { |
| struct stat statbuf; |
| BoolOrError ret(stat(path, &statbuf) == 0); |
| if (ret.is_false() && errno != ENOENT) { // Not error if file doesn't exist. |
| handler->Message(kError, "Failed to stat %s: %s", |
| path, strerror(errno)); |
| ret.set_error(); |
| } |
| return ret; |
| } |
| |
| BoolOrError StdioFileSystem::IsDir(const char* path, MessageHandler* handler) { |
| struct stat statbuf; |
| BoolOrError ret(false); |
| if (stat(path, &statbuf) == 0) { |
| #ifdef WIN32 |
| ret.set((statbuf.st_mode & _S_IFDIR) != 0); |
| #else |
| ret.set(S_ISDIR(statbuf.st_mode)); |
| #endif // WIN32 |
| } else if (errno != ENOENT) { // Not an error if file doesn't exist. |
| handler->Message(kError, "Failed to stat %s: %s", |
| path, strerror(errno)); |
| ret.set_error(); |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::ListContents(const StringPiece& dir, StringVector* files, |
| MessageHandler* handler) { |
| #ifdef WIN32 |
| const char kDirSeparator[] = "\\"; |
| std::string dir_string = dir.as_string(); |
| if (!dir.ends_with(kDirSeparator)) { |
| dir_string.append(kDirSeparator); |
| } |
| std::string pattern = dir_string + "*"; |
| std::wstring wpattern = std::wstring(pattern.begin(), pattern.end()); |
| WIN32_FIND_DATA entry; |
| HANDLE iter = FindFirstFile(wpattern.c_str(), &entry); |
| if (iter == INVALID_HANDLE_VALUE) { |
| handler->Error(dir_string.c_str(), 0, |
| "Failed to FindFirstFile: %s", strerror(errno)); |
| return false; |
| } |
| do { |
| std::wstring wfilename(entry.cFileName); |
| std::string filename(wfilename.begin(), wfilename.end()); // This is dodgy. |
| if (filename != "." && filename != "..") { |
| files->push_back(dir_string + filename); |
| } |
| } while (FindNextFile(iter, &entry) != 0); |
| if (GetLastError() != ERROR_NO_MORE_FILES) { |
| handler->Error(dir_string.c_str(), 0, |
| "Failed to FindNextFile: %s", strerror(errno)); |
| FindClose(iter); |
| return false; |
| } |
| FindClose(iter); |
| return true; |
| #else |
| GoogleString dir_string = dir.as_string(); |
| EnsureEndsInSlash(&dir_string); |
| const char* dirname = dir_string.c_str(); |
| DIR* mydir = opendir(dirname); |
| if (mydir == NULL) { |
| handler->Error(dirname, 0, "Failed to opendir: %s", strerror(errno)); |
| return false; |
| } else { |
| dirent* entry = NULL; |
| dirent buffer; |
| while (readdir_r(mydir, &buffer, &entry) == 0 && entry != NULL) { |
| if ((strcmp(entry->d_name, ".") != 0) && |
| (strcmp(entry->d_name, "..") != 0)) { |
| files->push_back(dir_string + entry->d_name); |
| } |
| } |
| if (closedir(mydir) != 0) { |
| handler->Error(dirname, 0, "Failed to closedir: %s", strerror(errno)); |
| return false; |
| } |
| return true; |
| } |
| #endif // WIN32 |
| } |
| |
| bool StdioFileSystem::Stat(const StringPiece& path, struct stat* statbuf, |
| MessageHandler* handler) { |
| const GoogleString path_string = path.as_string(); |
| const char* path_str = path_string.c_str(); |
| if (stat(path_str, statbuf) == 0) { |
| return true; |
| } else if (errno != ENOENT) { // Not an error if file doesn't exist see #972. |
| // https://github.com/pagespeed/ngx_pagespeed/issues/972 |
| handler->Message(kError, "Failed to stat %s: %s", path_str, |
| strerror(errno)); |
| } |
| return false; |
| } |
| |
| // TODO(abliss): there are some situations where this doesn't work |
| // -- e.g. if the filesystem is mounted noatime. We should try to |
| // detect that and provide a workaround. |
| bool StdioFileSystem::Atime(const StringPiece& path, int64* timestamp_sec, |
| MessageHandler* handler) { |
| struct stat statbuf; |
| bool ret = Stat(path, &statbuf, handler); |
| if (ret) { |
| *timestamp_sec = statbuf.st_atime; |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::Mtime(const StringPiece& path, int64* timestamp_sec, |
| MessageHandler* handler) { |
| struct stat statbuf; |
| bool ret = Stat(path, &statbuf, handler); |
| if (ret) { |
| *timestamp_sec = statbuf.st_mtime; |
| } |
| return ret; |
| } |
| |
| bool StdioFileSystem::Size(const StringPiece& path, int64* size, |
| MessageHandler* handler) { |
| struct stat statbuf; |
| bool ret = Stat(path, &statbuf, handler); |
| if (ret) { |
| #ifdef WIN32 |
| *size = statbuf.st_size; |
| #else |
| *size = statbuf.st_blocks * kBlockSize; |
| #endif // WIN32 |
| } |
| return ret; |
| } |
| |
| BoolOrError StdioFileSystem::TryLock(const StringPiece& lock_name, |
| MessageHandler* handler) { |
| const GoogleString lock_string = lock_name.as_string(); |
| const char* lock_str = lock_string.c_str(); |
| // POSIX mkdir is widely believed to be atomic, although I have |
| // found no reliable documentation of this fact. |
| #ifdef WIN32 |
| if (_mkdir(lock_str) == 0) { |
| #else |
| if (mkdir(lock_str, 0777) == 0) { |
| #endif // WIN32 |
| return BoolOrError(true); |
| } else if (errno == EEXIST) { |
| return BoolOrError(false); |
| } else { |
| handler->Message(kError, "Failed to mkdir %s: %s", |
| lock_str, strerror(errno)); |
| return BoolOrError(); |
| } |
| } |
| |
| BoolOrError StdioFileSystem::TryLockWithTimeout(const StringPiece& lock_name, |
| int64 timeout_ms, |
| const Timer* timer, |
| MessageHandler* handler) { |
| const GoogleString lock_string = lock_name.as_string(); |
| BoolOrError result = TryLock(lock_name, handler); |
| if (result.is_true() || result.is_error()) { |
| // We got the lock, or the lock is ungettable. |
| return result; |
| } |
| int64 m_time_sec; |
| if (!Mtime(lock_name, &m_time_sec, handler)) { |
| // We can't stat the lockfile. |
| return BoolOrError(); |
| } |
| |
| int64 now_us = timer->NowUs(); |
| int64 elapsed_since_lock_us = now_us - Timer::kSecondUs * m_time_sec; |
| int64 timeout_us = Timer::kMsUs * timeout_ms; |
| if (elapsed_since_lock_us < timeout_us) { |
| // The lock is held and timeout hasn't elapsed. |
| return BoolOrError(false); |
| } |
| // Lock has timed out. We have two options here: |
| // 1) Leave the lock in its present state and assume we've taken ownership. |
| // This is kind to the file system, but causes lots of repeated work at |
| // timeout, as subsequent threads also see a timed-out lock. |
| // 2) Force-unlock the lock and re-lock it. This resets the timeout period, |
| // but is hard on the file system metadata log. |
| const char* lock_str = lock_string.c_str(); |
| if (!Unlock(lock_name, handler)) { |
| // We couldn't break the lock. Maybe someone else beat us to it. |
| // We optimistically forge ahead anyhow (1), since we know we've timed out. |
| handler->Info(lock_str, 0, |
| "Breaking lock without reset! now-ctime=%d-%d > %d (sec)\n%s", |
| static_cast<int>(now_us / Timer::kSecondUs), |
| static_cast<int>(m_time_sec), |
| static_cast<int>(timeout_ms / Timer::kSecondMs), |
| StackTraceString().c_str()); |
| return BoolOrError(true); |
| } |
| handler->Info(lock_str, 0, "Broke lock! now-ctime=%d-%d > %d (sec)\n%s", |
| static_cast<int>(now_us / Timer::kSecondUs), |
| static_cast<int>(m_time_sec), |
| static_cast<int>(timeout_ms / Timer::kSecondMs), |
| StackTraceString().c_str()); |
| result = TryLock(lock_name, handler); |
| if (!result.is_true()) { |
| // Someone else grabbed the lock after we broke it. |
| handler->Info(lock_str, 0, "Failed to take lock after breaking it!"); |
| } |
| return result; |
| } |
| |
| bool StdioFileSystem::Unlock(const StringPiece& lock_name, |
| MessageHandler* handler) { |
| const GoogleString lock_string = lock_name.as_string(); |
| const char* lock_str = lock_string.c_str(); |
| #ifdef WIN32 |
| if (_rmdir(lock_str) == 0) { |
| #else |
| if (rmdir(lock_str) == 0) { |
| #endif // WIN32 |
| return true; |
| } else { |
| handler->Message(kError, "Failed to rmdir %s: %s", |
| lock_str, strerror(errno)); |
| return false; |
| } |
| } |
| |
| FileSystem::InputFile* StdioFileSystem::Stdin() { |
| return new StdioInputFile(stdin, "stdin", this); |
| } |
| |
| FileSystem::OutputFile* StdioFileSystem::Stdout() { |
| return new StdioOutputFile(stdout, "stdout", this); |
| } |
| |
| FileSystem::OutputFile* StdioFileSystem::Stderr() { |
| return new StdioOutputFile(stderr, "stderr", this); |
| } |
| |
| } // namespace net_instaweb |