From mboxrd@z Thu Jan 1 00:00:00 1970 From: Matthias Fischer To: development@lists.ipfire.org Subject: [PATCH] squid 3.5.20: latest patches from upstream Date: Sun, 24 Jul 2016 13:44:24 +0200 Message-ID: <20160724114425.28196-1-matthias.fischer@ipfire.org> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============2592185385001122821==" List-Id: --===============2592185385001122821== Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Signed-off-by: Matthias Fischer --- lfs/squid | 3 + src/patches/squid/squid-3.5-14067.patch | 381 ++++++++++++++++++++++++++++++= ++ src/patches/squid/squid-3.5-14068.patch | 35 +++ src/patches/squid/squid-3.5-14069.patch | 30 +++ 4 files changed, 449 insertions(+) create mode 100644 src/patches/squid/squid-3.5-14067.patch create mode 100644 src/patches/squid/squid-3.5-14068.patch create mode 100644 src/patches/squid/squid-3.5-14069.patch diff --git a/lfs/squid b/lfs/squid index 5bc976e..a88f0c5 100644 --- a/lfs/squid +++ b/lfs/squid @@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) : $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE) + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14067= .patch + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14068= .patch + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14069= .patch cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-3.5.17-fix-max-= file-descriptors.patch =20 cd $(DIR_APP) && autoreconf -vfi diff --git a/src/patches/squid/squid-3.5-14067.patch b/src/patches/squid/squi= d-3.5-14067.patch new file mode 100644 index 0000000..8d9cb21 --- /dev/null +++ b/src/patches/squid/squid-3.5-14067.patch @@ -0,0 +1,381 @@ +------------------------------------------------------------ +revno: 14067 +revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg +parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2 +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=3D4534 +committer: Amos Jeffries +branch nick: 3.5 +timestamp: Sat 2016-07-23 19:16:20 +1200 +message: + Bug 4534: assertion failure in xcalloc when using many cache_dir +------------------------------------------------------------ +# Bazaar merge directive format 2 (Bazaar 0.90) +# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363 +# timestamp: 2016-07-23 07:24:01 +0000 +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\ +# vpjak1pq4uecadd2 +#=20 +# Begin patch +=3D=3D=3D modified file 'src/CacheDigest.cc' +--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000 ++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000 +@@ -35,12 +35,12 @@ + static uint32_t hashed_keys[4]; +=20 + static void +-cacheDigestInit(CacheDigest * cd, int capacity, int bpe) ++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe) + { +- const size_t mask_size =3D cacheDigestCalcMaskSize(capacity, bpe); ++ const uint32_t mask_size =3D cacheDigestCalcMaskSize(capacity, bpe); + assert(cd); + assert(capacity > 0 && bpe > 0); +- assert(mask_size > 0); ++ assert(mask_size !=3D 0); + cd->capacity =3D capacity; + cd->bits_per_entry =3D bpe; + cd->mask_size =3D mask_size; +@@ -50,7 +50,7 @@ + } +=20 + CacheDigest * +-cacheDigestCreate(int capacity, int bpe) ++cacheDigestCreate(uint64_t capacity, uint8_t bpe) + { + CacheDigest *cd =3D (CacheDigest *)memAllocate(MEM_CACHE_DIGEST); + assert(SQUID_MD5_DIGEST_LENGTH =3D=3D 16); /* our hash functions rely = on 16 byte keys */ +@@ -97,7 +97,7 @@ +=20 + /* changes mask size, resets bits to 0, preserves "cd" pointer */ + void +-cacheDigestChangeCap(CacheDigest * cd, int new_cap) ++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap) + { + assert(cd); + cacheDigestClean(cd); +@@ -278,12 +278,12 @@ + storeAppendPrintf(e, "%s digest: size: %d bytes\n", + label ? label : "", stats.bit_count / 8 + ); +- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n", ++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu6= 4 " util: %d%%\n", + cd->count, + cd->capacity, + xpercentInt(cd->count, cd->capacity) + ); +- storeAppendPrintf(e, "\t deletion attempts: %d\n", ++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n", + cd->del_count + ); + storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: = %d%%\n", +@@ -297,16 +297,18 @@ + ); + } +=20 +-size_t +-cacheDigestCalcMaskSize(int cap, int bpe) ++uint32_t ++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe) + { +- return (size_t) (cap * bpe + 7) / 8; ++ uint64_t bitCount =3D (cap * bpe) + 7; ++ assert(bitCount < INT_MAX); // dont 31-bit overflow later ++ return static_cast(bitCount / 8); + } +=20 + static void + cacheDigestHashKey(const CacheDigest * cd, const cache_key * key) + { +- const unsigned int bit_count =3D cd->mask_size * 8; ++ const uint32_t bit_count =3D cd->mask_size * 8; + unsigned int tmp_keys[4]; + /* we must memcpy to ensure alignment */ + memcpy(tmp_keys, key, sizeof(tmp_keys)); + +=3D=3D=3D modified file 'src/CacheDigest.h' +--- src/CacheDigest.h 2016-01-01 00:14:27 +0000 ++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000 +@@ -22,23 +22,23 @@ + { + public: + /* public, read-only */ +- char *mask; /* bit mask */ +- int mask_size; /* mask size in bytes */ +- int capacity; /* expected maximum for .count, not a hard limit */ +- int bits_per_entry; /* number of bits allocated for each entry from= capacity */ +- int count; /* number of digested entries */ +- int del_count; /* number of deletions performed so far */ ++ uint64_t count; /* number of digested entries */ ++ uint64_t del_count; /* number of deletions performed so far */ ++ uint64_t capacity; /* expected maximum for .count, not a hard lim= it */ ++ char *mask; /* bit mask */ ++ uint32_t mask_size; /* mask size in bytes */ ++ int8_t bits_per_entry; /* number of bits allocated for each entry fro= m capacity */ + }; +=20 +-CacheDigest *cacheDigestCreate(int capacity, int bpe); ++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe); + void cacheDigestDestroy(CacheDigest * cd); + CacheDigest *cacheDigestClone(const CacheDigest * cd); + void cacheDigestClear(CacheDigest * cd); +-void cacheDigestChangeCap(CacheDigest * cd, int new_cap); ++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap); + int cacheDigestTest(const CacheDigest * cd, const cache_key * key); + void cacheDigestAdd(CacheDigest * cd, const cache_key * key); + void cacheDigestDel(CacheDigest * cd, const cache_key * key); +-size_t cacheDigestCalcMaskSize(int cap, int bpe); ++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe); + int cacheDigestBitUtil(const CacheDigest * cd); + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hi= t, int guess_hit); + void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats, Store= Entry * sentry, const char *label); + +=3D=3D=3D modified file 'src/PeerDigest.h' +--- src/PeerDigest.h 2016-01-01 00:14:27 +0000 ++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000 +@@ -52,7 +52,7 @@ + store_client *old_sc; + HttpRequest *request; + int offset; +- int mask_offset; ++ uint32_t mask_offset; + time_t start_time; + time_t resp_time; + time_t expires; + +=3D=3D=3D modified file 'src/peer_digest.cc' +--- src/peer_digest.cc 2016-01-01 00:14:27 +0000 ++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000 +@@ -754,7 +754,7 @@ + if (!reason && !size) { + if (!pd->cd) + reason =3D "null digest?!"; +- else if (fetch->mask_offset !=3D (int)pd->cd->mask_size) ++ else if (fetch->mask_offset !=3D pd->cd->mask_size) + reason =3D "premature end of digest?!"; + else if (!peerDigestUseful(pd)) + reason =3D "useless digest"; + +=3D=3D=3D modified file 'src/store_digest.cc' +--- src/store_digest.cc 2016-01-01 00:14:27 +0000 ++++ src/store_digest.cc 2016-07-23 07:16:20 +0000 +@@ -76,36 +76,63 @@ + static void storeDigestRewriteFinish(StoreEntry * e); + static EVH storeDigestSwapOutStep; + static void storeDigestCBlockSwapOut(StoreEntry * e); +-static int storeDigestCalcCap(void); +-static int storeDigestResize(void); + static void storeDigestAdd(const StoreEntry *); +=20 ++/// calculates digest capacity ++static uint64_t ++storeDigestCalcCap() ++{ ++ /* ++ * To-Do: Bloom proved that the optimal filter utilization is 50% (half= of ++ * the bits are off). However, we do not have a formula to calculate the ++ * number of _entries_ we want to pre-allocate for. ++ */ ++ const uint64_t hi_cap =3D Store::Root().maxSize() / Config.Store.avgObj= ectSize; ++ const uint64_t lo_cap =3D 1 + Store::Root().currentSize() / Config.Stor= e.avgObjectSize; ++ const uint64_t e_count =3D StoreEntry::inUseCount(); ++ uint64_t cap =3D e_count ? e_count : hi_cap; ++ debugs(71, 2, "have: " << e_count << ", want " << cap << ++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]"); ++ ++ if (cap < lo_cap) ++ cap =3D lo_cap; ++ ++ /* do not enforce hi_cap limit, average-based estimation may be wrong ++ *if (cap > hi_cap) ++ * cap =3D hi_cap; ++ */ ++ ++ // Bug 4534: we still have to set an upper-limit at some reasonable val= ue though. ++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX ++ const uint64_t absolute_max =3D (INT_MAX -8) / Config.digest.bits_per_e= ntry; ++ if (cap > absolute_max) { ++ static time_t last_loud =3D 0; ++ if (last_loud < squid_curtime - 86400) { ++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store "= << cap << " entries. Limiting to " << absolute_max); ++ last_loud =3D squid_curtime; ++ } else { ++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << "= entries. Limiting to " << absolute_max); ++ } ++ cap =3D absolute_max; ++ } ++ ++ return cap; ++} + #endif /* USE_CACHE_DIGESTS */ +=20 +-static void +-storeDigestRegisterWithCacheManager(void) ++void ++storeDigestInit(void) + { + Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, = 0, 1); +-} +- +-/* +- * PUBLIC FUNCTIONS +- */ +- +-void +-storeDigestInit(void) +-{ +- storeDigestRegisterWithCacheManager(); +=20 + #if USE_CACHE_DIGESTS +- const int cap =3D storeDigestCalcCap(); +- + if (!Config.onoff.digest_generation) { + store_digest =3D NULL; + debugs(71, 3, "Local cache digest generation disabled"); + return; + } +=20 ++ const uint64_t cap =3D storeDigestCalcCap(); + store_digest =3D cacheDigestCreate(cap, Config.digest.bits_per_entry); + debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite = every " << + (int) Config.digest.rebuild_period << "/" << +@@ -290,6 +317,31 @@ + storeDigestRebuildResume(); + } +=20 ++/// \returns true if we actually resized the digest ++static bool ++storeDigestResize() ++{ ++ const uint64_t cap =3D storeDigestCalcCap(); ++ assert(store_digest); ++ uint64_t diff; ++ if (cap > store_digest->capacity) ++ diff =3D cap - store_digest->capacity; ++ else ++ diff =3D store_digest->capacity - cap; ++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " << ++ diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)= " ); ++ /* avoid minor adjustments */ ++ ++ if (diff <=3D store_digest->capacity / 10) { ++ debugs(71, 2, "small change, will not resize."); ++ return false; ++ } else { ++ debugs(71, 2, "big change, resizing."); ++ cacheDigestChangeCap(store_digest, cap); ++ } ++ return true; ++} ++ + /* called be Rewrite to push Rebuild forward */ + static void + storeDigestRebuildResume(void) +@@ -439,7 +491,7 @@ + assert(e); + /* _add_ check that nothing bad happened while we were waiting @?@ @?@ = */ +=20 +- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size) ++ if (static_cast(sd_state.rewrite_offset + chunk_size) > store= _digest->mask_size) + chunk_size =3D store_digest->mask_size - sd_state.rewrite_offset; +=20 + e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size); +@@ -451,7 +503,7 @@ + sd_state.rewrite_offset +=3D chunk_size; +=20 + /* are we done ? */ +- if (sd_state.rewrite_offset >=3D store_digest->mask_size) ++ if (static_cast(sd_state.rewrite_offset) >=3D store_digest->m= ask_size) + storeDigestRewriteFinish(e); + else + eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.= 0, 1, false); +@@ -467,60 +519,10 @@ + sd_state.cblock.count =3D htonl(store_digest->count); + sd_state.cblock.del_count =3D htonl(store_digest->del_count); + sd_state.cblock.mask_size =3D htonl(store_digest->mask_size); +- sd_state.cblock.bits_per_entry =3D (unsigned char) +- Config.digest.bits_per_entry; ++ sd_state.cblock.bits_per_entry =3D Config.digest.bits_per_entry; + sd_state.cblock.hash_func_count =3D (unsigned char) CacheDigestHashFunc= Count; + e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock)); + } +=20 +-/* calculates digest capacity */ +-static int +-storeDigestCalcCap(void) +-{ +- /* +- * To-Do: Bloom proved that the optimal filter utilization is 50% (half= of +- * the bits are off). However, we do not have a formula to calculate the +- * number of _entries_ we want to pre-allocate for. +- */ +- const int hi_cap =3D Store::Root().maxSize() / Config.Store.avgObjectSi= ze; +- const int lo_cap =3D 1 + Store::Root().currentSize() / Config.Store.avg= ObjectSize; +- const int e_count =3D StoreEntry::inUseCount(); +- int cap =3D e_count ? e_count :hi_cap; +- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << c= ap << +- " entries; limits: [" << lo_cap << ", " << hi_cap << "]"); +- +- if (cap < lo_cap) +- cap =3D lo_cap; +- +- /* do not enforce hi_cap limit, average-based estimation may be wrong +- *if (cap > hi_cap) +- * cap =3D hi_cap; +- */ +- return cap; +-} +- +-/* returns true if we actually resized the digest */ +-static int +-storeDigestResize(void) +-{ +- const int cap =3D storeDigestCalcCap(); +- int diff; +- assert(store_digest); +- diff =3D abs(cap - store_digest->capacity); +- debugs(71, 2, "storeDigestResize: " << +- store_digest->capacity << " -> " << cap << "; change: " << +- diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)= " ); +- /* avoid minor adjustments */ +- +- if (diff <=3D store_digest->capacity / 10) { +- debugs(71, 2, "storeDigestResize: small change, will not resize."); +- return 0; +- } else { +- debugs(71, 2, "storeDigestResize: big change, resizing."); +- cacheDigestChangeCap(store_digest, cap); +- return 1; +- } +-} +- + #endif /* USE_CACHE_DIGESTS */ +=20 + +=3D=3D=3D modified file 'src/tests/stub_CacheDigest.cc' +--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000 ++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000 +@@ -16,11 +16,11 @@ + class CacheDigestGuessStats; + class StoreEntry; +=20 +-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL) ++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL) + void cacheDigestDestroy(CacheDigest *) STUB + CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL) + void cacheDigestClear(CacheDigest * ) STUB +-void cacheDigestChangeCap(CacheDigest *,int) STUB ++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB + int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1) + void cacheDigestAdd(CacheDigest *, const cache_key *) STUB + void cacheDigestDel(CacheDigest *, const cache_key *) STUB +@@ -28,5 +28,4 @@ + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry = *, const char *) STUB + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB +-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1) +- ++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1) + diff --git a/src/patches/squid/squid-3.5-14068.patch b/src/patches/squid/squi= d-3.5-14068.patch new file mode 100644 index 0000000..4766e00 --- /dev/null +++ b/src/patches/squid/squid-3.5-14068.patch @@ -0,0 +1,35 @@ +------------------------------------------------------------ +revno: 14068 +revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28 +parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=3D4542 +author: Anonymous +committer: Amos Jeffries +branch nick: 3.5 +timestamp: Sat 2016-07-23 19:19:30 +1200 +message: + Bug #4542: authentication credentials IP TTL updated incorrectly +------------------------------------------------------------ +# Bazaar merge directive format 2 (Bazaar 0.90) +# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28 +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8 +# timestamp: 2016-07-23 07:24:05 +0000 +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\ +# 1wzqpbyi1rk5w6vg +#=20 +# Begin patch +=3D=3D=3D modified file 'src/auth/User.cc' +--- src/auth/User.cc 2016-01-01 00:14:27 +0000 ++++ src/auth/User.cc 2016-07-23 07:19:30 +0000 +@@ -284,7 +284,7 @@ + /* This ip has already been seen. */ + found =3D 1; + /* update IP ttl */ +- ipdata->ip_expiretime =3D squid_curtime; ++ ipdata->ip_expiretime =3D squid_curtime + ::Config.authenticate= IpTTL; + } else if (ipdata->ip_expiretime <=3D squid_curtime) { + /* This IP has expired - remove from the seen list */ + dlinkDelete(&ipdata->node, &ip_list); + diff --git a/src/patches/squid/squid-3.5-14069.patch b/src/patches/squid/squi= d-3.5-14069.patch new file mode 100644 index 0000000..15ca37a --- /dev/null +++ b/src/patches/squid/squid-3.5-14069.patch @@ -0,0 +1,30 @@ +------------------------------------------------------------ +revno: 14069 +revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv +parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28 +committer: Source Maintenance +branch nick: 3.5 +timestamp: Sat 2016-07-23 12:13:51 +0000 +message: + SourceFormat Enforcement +------------------------------------------------------------ +# Bazaar merge directive format 2 (Bazaar 0.90) +# revision_id: squidadm(a)squid-cache.org-20160723121351-\ +# iuc8hwstrqd0l1dv +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28 +# timestamp: 2016-07-23 12:50:56 +0000 +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5 +# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\ +# cemledcltg8pkc28 +#=20 +# Begin patch +=3D=3D=3D modified file 'src/tests/stub_CacheDigest.cc' +--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000 ++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000 +@@ -29,3 +29,4 @@ + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry = *, const char *) STUB + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB + uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1) ++ + --=20 2.9.2 --===============2592185385001122821==--