Bug 3150: do not start useless unlinkd. Unlinkd may be used only by UFS storage. But before the change, Squid always started unlinkd if it is built, even if it is not needed. Whether a SwapDir may use unlinkd depends on the SwapDir implementation and DiskIO strategy it uses. The patch adds unlinkdUseful() method to SwapDir and DiskIOStrategy to decide if unlinkd should be started. After the change, unlinkd may be started during reconfiguration and unlinkdInit() may be called multiple times. After the change, unlinkdClose() may be called when unlinkd was never started. The patch removes a warning which was printed in this case on Windows. diff --git src/DiskIO/AIO/AIODiskIOStrategy.cc src/DiskIO/AIO/AIODiskIOStrategy.cc index 96e11f4..7a94637 100644 --- src/DiskIO/AIO/AIODiskIOStrategy.cc +++ src/DiskIO/AIO/AIODiskIOStrategy.cc @@ -83,40 +83,46 @@ AIODiskIOStrategy::newFile (char const *path) } return new AIODiskFile (path, this); } void AIODiskIOStrategy::sync() { assert(aq.aq_state == AQ_STATE_SETUP); /* * Keep calling callback to complete ops until the queue is empty * We can't quit when callback returns 0 - some calls may not * return any completed pending events, but they're still pending! */ while (aq.aq_numpending) callback(); } +bool +AIODiskIOStrategy::unlinkdUseful() const +{ + return false; +} + void AIODiskIOStrategy::unlinkFile (char const *) {} /* * Note: we grab the state and free the state before calling the callback * because this allows us to cut down the amount of time it'll take * to find a free slot (since if we call the callback first, we're going * to probably be allocated the slot _after_ this one..) * * I'll make it much more optimal later. */ int AIODiskIOStrategy::callback() { return 0; int i; int completed = 0; int retval, reterr; FREE *freefunc; diff --git src/DiskIO/AIO/AIODiskIOStrategy.h src/DiskIO/AIO/AIODiskIOStrategy.h index 9d461d0..c3c7d34 100644 --- src/DiskIO/AIO/AIODiskIOStrategy.h +++ src/DiskIO/AIO/AIODiskIOStrategy.h @@ -35,40 +35,42 @@ #if USE_DISKIO_AIO #include "DiskIO/DiskIOStrategy.h" #include "async_io.h" class AIODiskIOStrategy : public DiskIOStrategy { public: AIODiskIOStrategy(); virtual ~AIODiskIOStrategy(); virtual bool shedLoad(); /* What is the current load? 999 = 99.9% */ virtual int load(); /* Return a handle for performing IO operations */ virtual RefCount newFile (char const *path); /* flush all IO operations */ virtual void sync(); + /** whether the IO Strategy can use unlinkd */ + virtual bool unlinkdUseful() const; /* unlink a file by path */ virtual void unlinkFile (char const *); /* perform any pending callbacks */ virtual int callback(); /* Init per-instance logic */ virtual void init(); /* cachemgr output on the IO instance stats */ virtual void statfs(StoreEntry & sentry)const; /* module specific options */ virtual ConfigOption *getOptionTree() const; /* a file descriptor */ int fd; /* queue of requests */ async_queue_t aq; int findSlot(); }; diff --git src/DiskIO/Blocking/BlockingIOStrategy.cc src/DiskIO/Blocking/BlockingIOStrategy.cc index 9a61973..8610037 100644 --- src/DiskIO/Blocking/BlockingIOStrategy.cc +++ src/DiskIO/Blocking/BlockingIOStrategy.cc @@ -40,29 +40,35 @@ bool BlockingIOStrategy::shedLoad() { return false; } int BlockingIOStrategy::load() { /* Return 999 (99.9%) constant load */ return 999; } DiskFile::Pointer BlockingIOStrategy::newFile (char const *path) { return new BlockingFile (path); } +bool +BlockingIOStrategy::unlinkdUseful() const +{ + return true; +} + void BlockingIOStrategy::unlinkFile(char const *path) { #if USE_UNLINKD unlinkdUnlink(path); #else ::unlink(path); #endif } diff --git src/DiskIO/Blocking/BlockingIOStrategy.h src/DiskIO/Blocking/BlockingIOStrategy.h index 5dea6e2..8181dd1 100644 --- src/DiskIO/Blocking/BlockingIOStrategy.h +++ src/DiskIO/Blocking/BlockingIOStrategy.h @@ -28,24 +28,25 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * * Copyright (c) 2003, Robert Collins */ #ifndef SQUID_BLOCKINGIOSTRATEGY_H #define SQUID_BLOCKINGIOSTRATEGY_H #include "DiskIO/DiskIOStrategy.h" class BlockingIOStrategy : public DiskIOStrategy { public: virtual bool shedLoad(); virtual int load(); virtual RefCount newFile(char const *path); + virtual bool unlinkdUseful() const; virtual void unlinkFile (char const *); }; #endif /* SQUID_BLOCKINGIOSTRATEGY_H */ diff --git src/DiskIO/DiskDaemon/DiskdIOStrategy.cc src/DiskIO/DiskDaemon/DiskdIOStrategy.cc index 6d054e3..64c7012 100644 --- src/DiskIO/DiskDaemon/DiskdIOStrategy.cc +++ src/DiskIO/DiskDaemon/DiskdIOStrategy.cc @@ -86,40 +86,46 @@ DiskdIOStrategy::load() void DiskdIOStrategy::openFailed() { diskd_stats.open_fail_queue_len++; } DiskFile::Pointer DiskdIOStrategy::newFile(char const *path) { if (shedLoad()) { openFailed(); return NULL; } return new DiskdFile (path, this); } DiskdIOStrategy::DiskdIOStrategy() : magic1(64), magic2(72), away(0) , smsgid(-1), rmsgid(-1), wfd(-1) , instanceID(newInstance()) {} +bool +DiskdIOStrategy::unlinkdUseful() const +{ + return true; +} + void DiskdIOStrategy::unlinkFile(char const *path) { if (shedLoad()) { /* Damn, we need to issue a sync unlink here :( */ debugs(79, 2, "storeDiskUnlink: Out of queue space, sync unlink"); #if USE_UNLINKD unlinkdUnlink(path); #else unlink(path); #endif return; } /* We can attempt a diskd unlink */ int x; diff --git src/DiskIO/DiskDaemon/DiskdIOStrategy.h src/DiskIO/DiskDaemon/DiskdIOStrategy.h index 161d7ed..650400e 100644 --- src/DiskIO/DiskDaemon/DiskdIOStrategy.h +++ src/DiskIO/DiskDaemon/DiskdIOStrategy.h @@ -59,40 +59,41 @@ public: }; #include "DiskIO/DiskIOStrategy.h" #include "StoreIOState.h" class DiskFile; class DiskdFile; class ReadRequest; /// \ingroup diskd class DiskdIOStrategy : public DiskIOStrategy { public: DiskdIOStrategy(); virtual bool shedLoad(); virtual int load(); virtual RefCount newFile(char const *path); + virtual bool unlinkdUseful() const; virtual void unlinkFile (char const *); virtual ConfigOption *getOptionTree() const; virtual void init(); virtual void sync(); virtual int callback(); virtual void statfs(StoreEntry & sentry)const; int send(int mtype, int id, DiskdFile *theFile, size_t size, off_t offset, ssize_t shm_offset, RefCountable_ *requestor); /** public for accessing return address's */ SharedMemory shm; private: static size_t newInstance(); static size_t nextInstanceID; void openFailed(); bool optionQ1Parse(char const *option, const char *value, int reconfiguring); void optionQ1Dump(StoreEntry * e) const; bool optionQ2Parse(char const *option, const char *value, int reconfiguring); void optionQ2Dump(StoreEntry * e) const; int send(int mtype, int id, RefCount sio, size_t size, off_t offset, ssize_t shm_offset); diff --git src/DiskIO/DiskIOStrategy.h src/DiskIO/DiskIOStrategy.h index 2ec5617..91990d2 100644 --- src/DiskIO/DiskIOStrategy.h +++ src/DiskIO/DiskIOStrategy.h @@ -42,68 +42,73 @@ class DiskFile; class ConfigOption; class DiskIOStrategy { public: virtual ~DiskIOStrategy() {} /** Can the IO Strategy handle more requests ? */ virtual bool shedLoad() = 0; /** What is the current load? 999 = 99.9% */ virtual int load() = 0; /** Return a handle for performing IO operations */ virtual RefCount newFile(char const *path) = 0; /** flush all IO operations */ virtual void sync() {} + /** whether the IO Strategy can use unlinkd */ + virtual bool unlinkdUseful() const = 0; + /** unlink a file by path */ virtual void unlinkFile(char const *) = 0; /** perform any pending callbacks */ virtual int callback() { return 0; } /** Init per-instance logic */ virtual void init() {} /** cachemgr output on the IO instance stats */ virtual void statfs(StoreEntry & sentry)const {} /** module specific options */ virtual ConfigOption *getOptionTree() const { return NULL;} }; /* Because we need the DiskFile definition for newFile. */ #include "DiskFile.h" class SingletonIOStrategy : public DiskIOStrategy { public: SingletonIOStrategy(DiskIOStrategy *anIO) : io(anIO) {} virtual bool shedLoad() { return io->shedLoad(); } virtual int load() { return io->load(); } virtual RefCount newFile (char const *path) {return io->newFile(path); } virtual void sync() { io->sync(); } + virtual bool unlinkdUseful() const { return io->unlinkdUseful(); } + virtual void unlinkFile (char const *path) { io->unlinkFile(path); } virtual int callback() { return io->callback(); } virtual void init() { io->init(); } virtual void statfs(StoreEntry & sentry)const { io->statfs(sentry); } virtual ConfigOption *getOptionTree() const { return io->getOptionTree(); } private: DiskIOStrategy *io; }; #endif /* SQUID_DISKIOSTRATEGY_H */ diff --git src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc index e7c0fd9..1ba1ecf 100644 --- src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc +++ src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc @@ -237,26 +237,32 @@ DiskThreadsIOStrategy::load() if (ql == 0) loadav = 0; loadav = ql * 1000 / MAGIC1; debugs(47, 9, "DiskThreadsIOStrategy::load: load=" << loadav); return loadav; } DiskFile::Pointer DiskThreadsIOStrategy::newFile (char const *path) { if (shedLoad()) { return NULL; } return new DiskThreadsDiskFile (path, this); } +bool +DiskThreadsIOStrategy::unlinkdUseful() const +{ + return false; +} + void DiskThreadsIOStrategy::unlinkFile(char const *path) { statCounter.syscalls.disk.unlinks++; aioUnlink(path, NULL, NULL); } diff --git src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h index 12da076..d7e54b1 100644 --- src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h +++ src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h @@ -37,36 +37,37 @@ #ifndef __STORE_DISKTHREADEDIOSTRATEGY_H__ #define __STORE_DISKTHREADEDIOSTRATEGY_H__ #define _AIO_OPEN 0 #define _AIO_READ 1 #define _AIO_WRITE 2 #define _AIO_CLOSE 3 #define _AIO_UNLINK 4 #define _AIO_OPENDIR 5 #define _AIO_STAT 6 #include "DiskIO/DiskIOStrategy.h" class DiskThreadsIOStrategy : public DiskIOStrategy { public: DiskThreadsIOStrategy(); virtual bool shedLoad(); virtual int load(); virtual RefCount newFile(char const *path); + virtual bool unlinkdUseful() const; virtual void unlinkFile (char const *); virtual int callback(); virtual void sync(); virtual void init(); void done(); /* Todo: add access limitations */ bool initialised; static DiskThreadsIOStrategy Instance; MemAllocator *squidaio_ctrl_pool; private: static void aioStats(StoreEntry * sentry); void registerWithCacheManager(void); }; #endif diff --git src/DiskIO/IpcIo/IpcIoIOStrategy.cc src/DiskIO/IpcIo/IpcIoIOStrategy.cc index 10a165e..dc3a191 100644 --- src/DiskIO/IpcIo/IpcIoIOStrategy.cc +++ src/DiskIO/IpcIo/IpcIoIOStrategy.cc @@ -11,29 +11,35 @@ bool IpcIoIOStrategy::shedLoad() { return false; } int IpcIoIOStrategy::load() { /* Return 999 (99.9%) constant load */ return 999; } DiskFile::Pointer IpcIoIOStrategy::newFile (char const *path) { return new IpcIoFile (path); } +bool +IpcIoIOStrategy::unlinkdUseful() const +{ + return true; +} + void IpcIoIOStrategy::unlinkFile(char const *path) { #if USE_UNLINKD unlinkdUnlink(path); #else ::unlink(path); #endif } diff --git src/DiskIO/IpcIo/IpcIoIOStrategy.h src/DiskIO/IpcIo/IpcIoIOStrategy.h index 3bd2945..447e8e3 100644 --- src/DiskIO/IpcIo/IpcIoIOStrategy.h +++ src/DiskIO/IpcIo/IpcIoIOStrategy.h @@ -1,15 +1,16 @@ #ifndef SQUID_IPC_IOIOSTRATEGY_H #define SQUID_IPC_IOIOSTRATEGY_H #include "DiskIO/DiskIOStrategy.h" class IpcIoIOStrategy : public DiskIOStrategy { public: virtual bool shedLoad(); virtual int load(); virtual RefCount newFile(char const *path); + virtual bool unlinkdUseful() const; virtual void unlinkFile (char const *); }; #endif /* SQUID_IPC_IOIOSTRATEGY_H */ diff --git src/DiskIO/Mmapped/MmappedIOStrategy.cc src/DiskIO/Mmapped/MmappedIOStrategy.cc index 6f01590..3f6d681 100644 --- src/DiskIO/Mmapped/MmappedIOStrategy.cc +++ src/DiskIO/Mmapped/MmappedIOStrategy.cc @@ -11,29 +11,35 @@ bool MmappedIOStrategy::shedLoad() { return false; } int MmappedIOStrategy::load() { /* Return 999 (99.9%) constant load */ return 999; } DiskFile::Pointer MmappedIOStrategy::newFile (char const *path) { return new MmappedFile (path); } +bool +MmappedIOStrategy::unlinkdUseful() const +{ + return true; +} + void MmappedIOStrategy::unlinkFile(char const *path) { #if USE_UNLINKD unlinkdUnlink(path); #else ::unlink(path); #endif } diff --git src/DiskIO/Mmapped/MmappedIOStrategy.h src/DiskIO/Mmapped/MmappedIOStrategy.h index c295427..2d17524 100644 --- src/DiskIO/Mmapped/MmappedIOStrategy.h +++ src/DiskIO/Mmapped/MmappedIOStrategy.h @@ -1,15 +1,16 @@ #ifndef SQUID_MMAPPEDIOSTRATEGY_H #define SQUID_MMAPPEDIOSTRATEGY_H #include "DiskIO/DiskIOStrategy.h" class MmappedIOStrategy : public DiskIOStrategy { public: virtual bool shedLoad(); virtual int load(); virtual RefCount newFile(char const *path); + virtual bool unlinkdUseful() const; virtual void unlinkFile (char const *); }; #endif /* SQUID_MMAPPEDIOSTRATEGY_H */ diff --git src/SwapDir.h src/SwapDir.h index 723443c..00d7a03 100644 --- src/SwapDir.h +++ src/SwapDir.h @@ -115,40 +115,42 @@ SQUIDCEXTERN void storeDirLRUDelete(StoreEntry *); SQUIDCEXTERN void storeDirLRUAdd(StoreEntry *); SQUIDCEXTERN int storeDirGetBlkSize(const char *path, int *blksize); SQUIDCEXTERN int storeDirGetUFSStats(const char *, int *, int *, int *, int *); /// manages a single cache_dir class SwapDir : public Store { public: typedef RefCount Pointer; SwapDir(char const *aType); virtual ~SwapDir(); virtual void reconfigure() = 0; char const *type() const; virtual bool needsDiskStrand() const; ///< needs a dedicated kid process virtual bool active() const; ///< may be used in this strand /// whether stat should be reported by this SwapDir virtual bool doReportStat() const { return active(); } + /// whether SwapDir may benefit from unlinkd + virtual bool unlinkdUseful() const = 0; /* official Store interface functions */ virtual void diskFull(); virtual StoreEntry * get(const cache_key *); virtual void get(String const, STOREGETCLIENT, void * cbdata); virtual uint64_t maxSize() const { return max_size;} virtual uint64_t minSize() const; virtual int64_t maxObjectSize() const { return max_objsize; } virtual void stat (StoreEntry &anEntry) const; virtual StoreSearch *search(String const url, HttpRequest *) = 0; /* migrated from store_dir.cc */ bool objectSizeIsAcceptable(int64_t objsize) const; diff --git src/fs/coss/CossSwapDir.h src/fs/coss/CossSwapDir.h index 6929d81..8279d27 100644 --- src/fs/coss/CossSwapDir.h +++ src/fs/coss/CossSwapDir.h @@ -18,40 +18,41 @@ class DiskFile; #endif /* Note that swap_filen in sio/e are actually disk offsets too! */ /* What we're doing in storeCossAllocate() */ #define COSS_ALLOC_ALLOCATE 1 #define COSS_ALLOC_REALLOC 2 /// \ingroup COSS class CossSwapDir : public SwapDir, public IORequestor { public: CossSwapDir(); virtual void init(); virtual void create(); virtual void dump(StoreEntry &)const; ~CossSwapDir(); virtual StoreSearch *search(String const url, HttpRequest *); + virtual bool unlinkdUseful() const; virtual void unlink (StoreEntry &); virtual void statfs (StoreEntry &)const; virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; virtual int callback(); virtual void sync(); virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual void openLog(); virtual void closeLog(); virtual int writeCleanStart(); virtual void writeCleanDone(); virtual void logEntry(const StoreEntry & e, int op) const; virtual void parse (int index, char *path); virtual void reconfigure(); virtual void swappedOut(const StoreEntry &e); virtual uint64_t currentSize() const { return cur_size; } virtual uint64_t currentCount() const { return n_disk_objects; } /* internals */ virtual off_t storeCossFilenoToDiskOffset(sfileno); virtual sfileno storeCossDiskOffsetToFileno(off_t); diff --git src/fs/coss/store_io_coss.cc src/fs/coss/store_io_coss.cc index bb4c09d..8019d13 100644 --- src/fs/coss/store_io_coss.cc +++ src/fs/coss/store_io_coss.cc @@ -112,40 +112,47 @@ CossSwapDir::allocate(const StoreEntry * e, int which) current_membuf = newmb; } /* * If we didn't get a collision, then update the current offset * and return it */ if (coll == 0) { retofs = current_offset; current_offset = retofs + allocsize; /* round up to our blocksize */ current_offset = ((current_offset + blksz_mask) >> blksz_bits ) << blksz_bits; return storeCossDiskOffsetToFileno(retofs); } else { StoreFScoss::GetInstance().stats.alloc.collisions++; debugs(79, 3, "CossSwapDir::allocate: Collision"); return -1; } } +bool +CossSwapDir::unlinkdUseful() const +{ + // no entry-specific files to unlink + return false; +} + void CossSwapDir::unlink(StoreEntry & e) { debugs(79, 3, "storeCossUnlink: offset " << e.swap_filen); if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) { cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); --n_disk_objects; } StoreFScoss::GetInstance().stats.unlink.ops++; StoreFScoss::GetInstance().stats.unlink.success++; storeCossRemove(this, &e); } StoreIOState::Pointer CossSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * callback, void *callback_data) { CossState *cstate; StoreIOState::Pointer sio = new CossState(this); cstate = dynamic_cast(sio.getRaw()); sio->offset_ = 0; diff --git src/fs/rock/RockSwapDir.cc src/fs/rock/RockSwapDir.cc index 8924d43..6afea2c 100644 --- src/fs/rock/RockSwapDir.cc +++ src/fs/rock/RockSwapDir.cc @@ -727,40 +727,47 @@ Rock::SwapDir::maintain() void Rock::SwapDir::reference(StoreEntry &e) { debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen); if (repl && repl->Referenced) repl->Referenced(repl, &e, &e.repl); } bool Rock::SwapDir::dereference(StoreEntry &e) { debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen); if (repl && repl->Dereferenced) repl->Dereferenced(repl, &e, &e.repl); // no need to keep e in the global store_table for us; we have our own map return false; } +bool +Rock::SwapDir::unlinkdUseful() const +{ + // no entry-specific files to unlink + return false; +} + void Rock::SwapDir::unlink(StoreEntry &e) { debugs(47, 5, HERE << e); ignoreReferences(e); map->free(e.swap_filen); disconnect(e); } void Rock::SwapDir::trackReferences(StoreEntry &e) { debugs(47, 5, HERE << e); if (repl) repl->Add(repl, &e, &e.repl); } void Rock::SwapDir::ignoreReferences(StoreEntry &e) diff --git src/fs/rock/RockSwapDir.h src/fs/rock/RockSwapDir.h index e2b8e37..b05d1e7 100644 --- src/fs/rock/RockSwapDir.h +++ src/fs/rock/RockSwapDir.h @@ -36,40 +36,41 @@ public: int64_t entryLimitHigh() const { return SwapFilenMax; } ///< Core limit int64_t entryLimitAllowed() const; typedef Ipc::StoreMapWithExtras DirMap; protected: /* protected ::SwapDir API */ virtual bool needsDiskStrand() const; virtual void create(); virtual void init(); virtual ConfigOption *getOptionTree() const; virtual bool allowOptionReconfigure(const char *const option) const; virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual void maintain(); virtual void diskFull(); virtual void reference(StoreEntry &e); virtual bool dereference(StoreEntry &e); + virtual bool unlinkdUseful() const; virtual void unlink(StoreEntry &e); virtual void statfs(StoreEntry &e) const; /* IORequestor API */ virtual void ioCompletedNotification(); virtual void closeCompleted(); virtual void readCompleted(const char *buf, int len, int errflag, RefCount< ::ReadRequest>); virtual void writeCompleted(int errflag, size_t len, RefCount< ::WriteRequest>); virtual void parse(int index, char *path); void parseSize(const bool reconfiguring); ///< parses anonymous cache_dir size option void validateOptions(); ///< warns of configuration problems; may quit bool parseTimeOption(char const *option, const char *value, int reconfiguring); void dumpTimeOption(StoreEntry * e) const; bool parseRateOption(char const *option, const char *value, int reconfiguring); void dumpRateOption(StoreEntry * e) const; void rebuild(); ///< starts loading and validating stored entry metadata ///< used to add entries successfully loaded during rebuild bool addEntry(const int fileno, const DbCellHeader &header, const StoreEntry &from); diff --git src/fs/ufs/store_dir_ufs.cc src/fs/ufs/store_dir_ufs.cc index 129629c..7ee1df8 100644 --- src/fs/ufs/store_dir_ufs.cc +++ src/fs/ufs/store_dir_ufs.cc @@ -1270,40 +1270,47 @@ UFSSwapDir::validFileno(sfileno filn, int flag) const /* * UFSSwapDir::unlinkFile * * This routine unlinks a file and pulls it out of the bitmap. * It used to be in commonUfsUnlink(), however an interface change * forced this bit of code here. Eeek. */ void UFSSwapDir::unlinkFile(sfileno f) { debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') << std::hex << std::uppercase << std::setw(8) << f << " '" << fullPath(f,NULL) << "'"); /* commonUfsDirMapBitReset(this, f); */ IO->unlinkFile(fullPath(f,NULL)); } +bool +UFSSwapDir::unlinkdUseful() const +{ + // unlinkd may be useful only in workers + return IamWorkerProcess() && IO->io->unlinkdUseful(); +} + void UFSSwapDir::unlink(StoreEntry & e) { debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) { cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); --n_disk_objects; } replacementRemove(&e); mapBitReset(e.swap_filen); UFSSwapDir::unlinkFile(e.swap_filen); } /* * Add and remove the given StoreEntry from the replacement policy in * use. */ void diff --git src/fs/ufs/ufscommon.h src/fs/ufs/ufscommon.h index 7561e8b..3af0619 100644 --- src/fs/ufs/ufscommon.h +++ src/fs/ufs/ufscommon.h @@ -42,40 +42,41 @@ class DiskIOModule; class StoreSearch; #include "SwapDir.h" /// \ingroup UFS class UFSSwapDir : public SwapDir { public: static int IsUFSDir(SwapDir* sd); static int DirClean(int swap_index); static int FilenoBelongsHere(int fn, int F0, int F1, int F2); UFSSwapDir(char const *aType, const char *aModuleType); virtual void init(); virtual void create(); virtual void dump(StoreEntry &) const; ~UFSSwapDir(); virtual StoreSearch *search(String const url, HttpRequest *); virtual bool doubleCheck(StoreEntry &); + virtual bool unlinkdUseful() const; virtual void unlink(StoreEntry &); virtual void statfs(StoreEntry &)const; virtual void maintain(); virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; virtual void reference(StoreEntry &); virtual bool dereference(StoreEntry &); virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual void openLog(); virtual void closeLog(); virtual int writeCleanStart(); virtual void writeCleanDone(); virtual void logEntry(const StoreEntry & e, int op) const; virtual void parse(int index, char *path); virtual void reconfigure(); virtual int callback(); virtual void sync(); virtual void swappedOut(const StoreEntry &e); virtual uint64_t currentSize() const { return cur_size; } virtual uint64_t currentCount() const { return n_disk_objects; } diff --git src/main.cc src/main.cc index 546eb92..56eadc4 100644 --- src/main.cc +++ src/main.cc @@ -855,40 +855,45 @@ mainReconfigureFinish(void *) if (IamPrimaryProcess()) { #if USE_WCCP wccpInit(); #endif #if USE_WCCPv2 wccp2Init(); #endif } serverConnectionsOpen(); neighbors_init(); storeDirOpenSwapLogs(); mimeInit(Config.mimeTablePathname); +#if USE_UNLINKD + if (unlinkdNeeded()) + unlinkdInit(); +#endif + #if USE_DELAY_POOLS Config.ClientDelay.finalize(); #endif if (Config.onoff.announce) { if (!eventFind(start_announce, NULL)) eventAdd("start_announce", start_announce, NULL, 3600.0, 1); } else { if (eventFind(start_announce, NULL)) eventDelete(start_announce, NULL); } writePidFile(); /* write PID file */ debugs(1, 1, "Ready to serve requests."); reconfiguring = 0; } static void @@ -1068,41 +1073,42 @@ mainInitialize(void) icapLogOpen(); #endif #if USE_IDENT Ident::Init(); #endif #if SQUID_SNMP snmpInit(); #endif #if MALLOC_DBG malloc_debug(0, malloc_debug_level); #endif if (!configured_once) { #if USE_UNLINKD - unlinkdInit(); + if (unlinkdNeeded()) + unlinkdInit(); #endif urlInitialize(); statInit(); storeInit(); mainSetCwd(); /* after this point we want to see the mallinfo() output */ do_mallinfo = 1; mimeInit(Config.mimeTablePathname); refreshInit(); #if USE_DELAY_POOLS DelayPools::Init(); #endif FwdState::initModule(); /* register the modules in the cache manager menus */ cbdataRegisterWithCacheManager(); /* These use separate calls so that the comm loops can eventually * coexist. diff --git src/protos.h src/protos.h index 46cb7b8..00bae33 100644 --- src/protos.h +++ src/protos.h @@ -574,40 +574,41 @@ SQUIDCEXTERN int DebugSignal; /* AYJ debugs function to show locations being reset with memset() */ SQUIDCEXTERN void *xmemset(void *dst, int, size_t); SQUIDCEXTERN void debug_trap(const char *); SQUIDCEXTERN void logsFlush(void); SQUIDCEXTERN const char *checkNullString(const char *p); SQUIDCEXTERN void squid_getrusage(struct rusage *r); SQUIDCEXTERN double rusage_cputime(struct rusage *r); SQUIDCEXTERN int rusage_maxrss(struct rusage *r); SQUIDCEXTERN int rusage_pagefaults(struct rusage *r); SQUIDCEXTERN void releaseServerSockets(void); SQUIDCEXTERN void PrintRusage(void); SQUIDCEXTERN void dumpMallocStats(void); #if USE_UNLINKD +SQUIDCEXTERN bool unlinkdNeeded(void); SQUIDCEXTERN void unlinkdInit(void); SQUIDCEXTERN void unlinkdClose(void); SQUIDCEXTERN void unlinkdUnlink(const char *); #endif SQUIDCEXTERN AnyP::ProtocolType urlParseProtocol(const char *, const char *e = NULL); SQUIDCEXTERN void urlInitialize(void); SQUIDCEXTERN HttpRequest *urlParse(const HttpRequestMethod&, char *, HttpRequest *request = NULL); SQUIDCEXTERN const char *urlCanonical(HttpRequest *); SQUIDCEXTERN char *urlCanonicalClean(const HttpRequest *); SQUIDCEXTERN const char *urlCanonicalFakeHttps(const HttpRequest * request); SQUIDCEXTERN bool urlIsRelative(const char *); SQUIDCEXTERN char *urlMakeAbsolute(const HttpRequest *, const char *); SQUIDCEXTERN char *urlRInternal(const char *host, unsigned short port, const char *dir, const char *name); SQUIDCEXTERN char *urlInternal(const char *dir, const char *name); SQUIDCEXTERN int matchDomainName(const char *host, const char *domain); SQUIDCEXTERN int urlCheckRequest(const HttpRequest *); SQUIDCEXTERN int urlDefaultPort(AnyP::ProtocolType p); SQUIDCEXTERN char *urlHostname(const char *url); SQUIDCEXTERN void urlExtMethodConfigure(void); diff --git src/tests/TestSwapDir.cc src/tests/TestSwapDir.cc index d99d28b..b650b64 100644 --- src/tests/TestSwapDir.cc +++ src/tests/TestSwapDir.cc @@ -19,40 +19,46 @@ uint64_t TestSwapDir::currentCount() const { return 2; } void TestSwapDir::stat(StoreEntry &) const { const_cast(this)->statsCalled = true; } void TestSwapDir::reconfigure() {} void TestSwapDir::init() {} bool +TestSwapDir::unlinkdUseful() const +{ + return false; +} + +bool TestSwapDir::canStore(const StoreEntry &, int64_t, int &load) const { load = 0; return true; } StoreIOState::Pointer TestSwapDir::createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) { return NULL; } StoreIOState::Pointer TestSwapDir::openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) { return NULL; } void TestSwapDir::parse(int, char*) diff --git src/tests/TestSwapDir.h src/tests/TestSwapDir.h index e5d51cb..ad6f81d 100644 --- src/tests/TestSwapDir.h +++ src/tests/TestSwapDir.h @@ -3,30 +3,31 @@ #include "squid.h" #include "SwapDir.h" class TestSwapDir : public SwapDir { public: TestSwapDir() : SwapDir("test"), statsCalled (false) {} bool statsCalled; virtual uint64_t maxSize() const; virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */ virtual void swappedOut(const StoreEntry &e) {} virtual void reconfigure(); virtual void init(); + virtual bool unlinkdUseful() const; virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); virtual void parse(int, char*); virtual StoreSearch *search(String, HttpRequest *); }; typedef RefCount TestSwapDirPointer; #endif /* TEST_TESTSWAPDIR */ diff --git src/unlinkd.cc src/unlinkd.cc index c46e1c2..09066fb 100644 --- src/unlinkd.cc +++ src/unlinkd.cc @@ -18,40 +18,41 @@ * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" #include "SquidTime.h" +#include "SwapDir.h" #include "fde.h" #include "xusleep.h" /* This code gets linked to Squid */ static int unlinkd_wfd = -1; static int unlinkd_rfd = -1; static void * hIpc; static pid_t pid; #define UNLINKD_QUEUE_LIMIT 20 void unlinkdUnlink(const char *path) { char buf[MAXPATHLEN]; int l; int bytes_written; static int queuelen = 0; @@ -139,75 +140,90 @@ unlinkdUnlink(const char *path) statCounter.syscalls.disk.unlinks++; queuelen++; } void unlinkdClose(void) #if _SQUID_MSWIN_ { if (unlinkd_wfd > -1) { debugs(2, 1, "Closing unlinkd pipe on FD " << unlinkd_wfd); shutdown(unlinkd_wfd, SD_BOTH); comm_close(unlinkd_wfd); if (unlinkd_wfd != unlinkd_rfd) comm_close(unlinkd_rfd); unlinkd_wfd = -1; unlinkd_rfd = -1; - } else - debugs(2, 0, "unlinkdClose: WARNING: unlinkd_wfd is " << unlinkd_wfd); + } if (hIpc) { if (WaitForSingleObject(hIpc, 5000) != WAIT_OBJECT_0) { getCurrentTime(); debugs(2, 1, "unlinkdClose: WARNING: (unlinkd," << pid << "d) didn't exit in 5 seconds"); } CloseHandle(hIpc); } } #else { if (unlinkd_wfd < 0) return; debugs(2, 1, "Closing unlinkd pipe on FD " << unlinkd_wfd); file_close(unlinkd_wfd); if (unlinkd_wfd != unlinkd_rfd) file_close(unlinkd_rfd); unlinkd_wfd = -1; unlinkd_rfd = -1; } #endif +bool +unlinkdNeeded(void) +{ + // we should start unlinkd if there are any cache_dirs using it + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) { + const RefCount sd = Config.cacheSwap.swapDirs[i]; + if (sd->unlinkdUseful()) + return true; + } + + return false; +} + void unlinkdInit(void) { + if (unlinkd_wfd >= 0) + return; // unlinkd already started + const char *args[2]; Ip::Address localhost; args[0] = "(unlinkd)"; args[1] = NULL; localhost.SetLocalhost(); pid = ipcCreate( #if USE_POLL && defined(_SQUID_OSF_) /* pipes and poll() don't get along on DUNIX -DW */ IPC_STREAM, #elif defined(_SQUID_MSWIN_) /* select() will fail on a pipe */ IPC_TCP_SOCKET, #else /* We currently need to use FIFO.. see below */ IPC_FIFO, #endif Config.Program.unlinkd, args,