* [PATCH] squid 3.5.20: latest patches from upstream
@ 2016-07-24 11:44 Matthias Fischer
2016-08-02 15:14 ` Michael Tremer
0 siblings, 1 reply; 7+ messages in thread
From: Matthias Fischer @ 2016-07-24 11:44 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 20182 bytes --]
Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
---
lfs/squid | 3 +
src/patches/squid/squid-3.5-14067.patch | 381 ++++++++++++++++++++++++++++++++
src/patches/squid/squid-3.5-14068.patch | 35 +++
src/patches/squid/squid-3.5-14069.patch | 30 +++
4 files changed, 449 insertions(+)
create mode 100644 src/patches/squid/squid-3.5-14067.patch
create mode 100644 src/patches/squid/squid-3.5-14068.patch
create mode 100644 src/patches/squid/squid-3.5-14069.patch
diff --git a/lfs/squid b/lfs/squid
index 5bc976e..a88f0c5 100644
--- a/lfs/squid
+++ b/lfs/squid
@@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
$(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
@$(PREBUILD)
@rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE)
+ cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14067.patch
+ cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14068.patch
+ cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14069.patch
cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-3.5.17-fix-max-file-descriptors.patch
cd $(DIR_APP) && autoreconf -vfi
diff --git a/src/patches/squid/squid-3.5-14067.patch b/src/patches/squid/squid-3.5-14067.patch
new file mode 100644
index 0000000..8d9cb21
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14067.patch
@@ -0,0 +1,381 @@
+------------------------------------------------------------
+revno: 14067
+revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
+fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
+committer: Amos Jeffries <squid3(a)treenet.co.nz>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 19:16:20 +1200
+message:
+ Bug 4534: assertion failure in xcalloc when using many cache_dir
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
+# timestamp: 2016-07-23 07:24:01 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
+# vpjak1pq4uecadd2
+#
+# Begin patch
+=== modified file 'src/CacheDigest.cc'
+--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
+@@ -35,12 +35,12 @@
+ static uint32_t hashed_keys[4];
+
+ static void
+-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
+ {
+- const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
++ const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
+ assert(cd);
+ assert(capacity > 0 && bpe > 0);
+- assert(mask_size > 0);
++ assert(mask_size != 0);
+ cd->capacity = capacity;
+ cd->bits_per_entry = bpe;
+ cd->mask_size = mask_size;
+@@ -50,7 +50,7 @@
+ }
+
+ CacheDigest *
+-cacheDigestCreate(int capacity, int bpe)
++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
+ {
+ CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
+ assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely on 16 byte keys */
+@@ -97,7 +97,7 @@
+
+ /* changes mask size, resets bits to 0, preserves "cd" pointer */
+ void
+-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
+ {
+ assert(cd);
+ cacheDigestClean(cd);
+@@ -278,12 +278,12 @@
+ storeAppendPrintf(e, "%s digest: size: %d bytes\n",
+ label ? label : "", stats.bit_count / 8
+ );
+- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64 " util: %d%%\n",
+ cd->count,
+ cd->capacity,
+ xpercentInt(cd->count, cd->capacity)
+ );
+- storeAppendPrintf(e, "\t deletion attempts: %d\n",
++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
+ cd->del_count
+ );
+ storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: %d%%\n",
+@@ -297,16 +297,18 @@
+ );
+ }
+
+-size_t
+-cacheDigestCalcMaskSize(int cap, int bpe)
++uint32_t
++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
+ {
+- return (size_t) (cap * bpe + 7) / 8;
++ uint64_t bitCount = (cap * bpe) + 7;
++ assert(bitCount < INT_MAX); // dont 31-bit overflow later
++ return static_cast<uint32_t>(bitCount / 8);
+ }
+
+ static void
+ cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
+ {
+- const unsigned int bit_count = cd->mask_size * 8;
++ const uint32_t bit_count = cd->mask_size * 8;
+ unsigned int tmp_keys[4];
+ /* we must memcpy to ensure alignment */
+ memcpy(tmp_keys, key, sizeof(tmp_keys));
+
+=== modified file 'src/CacheDigest.h'
+--- src/CacheDigest.h 2016-01-01 00:14:27 +0000
++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
+@@ -22,23 +22,23 @@
+ {
+ public:
+ /* public, read-only */
+- char *mask; /* bit mask */
+- int mask_size; /* mask size in bytes */
+- int capacity; /* expected maximum for .count, not a hard limit */
+- int bits_per_entry; /* number of bits allocated for each entry from capacity */
+- int count; /* number of digested entries */
+- int del_count; /* number of deletions performed so far */
++ uint64_t count; /* number of digested entries */
++ uint64_t del_count; /* number of deletions performed so far */
++ uint64_t capacity; /* expected maximum for .count, not a hard limit */
++ char *mask; /* bit mask */
++ uint32_t mask_size; /* mask size in bytes */
++ int8_t bits_per_entry; /* number of bits allocated for each entry from capacity */
+ };
+
+-CacheDigest *cacheDigestCreate(int capacity, int bpe);
++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
+ void cacheDigestDestroy(CacheDigest * cd);
+ CacheDigest *cacheDigestClone(const CacheDigest * cd);
+ void cacheDigestClear(CacheDigest * cd);
+-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
+ int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
+ void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
+ void cacheDigestDel(CacheDigest * cd, const cache_key * key);
+-size_t cacheDigestCalcMaskSize(int cap, int bpe);
++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
+ int cacheDigestBitUtil(const CacheDigest * cd);
+ void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hit, int guess_hit);
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats, StoreEntry * sentry, const char *label);
+
+=== modified file 'src/PeerDigest.h'
+--- src/PeerDigest.h 2016-01-01 00:14:27 +0000
++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
+@@ -52,7 +52,7 @@
+ store_client *old_sc;
+ HttpRequest *request;
+ int offset;
+- int mask_offset;
++ uint32_t mask_offset;
+ time_t start_time;
+ time_t resp_time;
+ time_t expires;
+
+=== modified file 'src/peer_digest.cc'
+--- src/peer_digest.cc 2016-01-01 00:14:27 +0000
++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
+@@ -754,7 +754,7 @@
+ if (!reason && !size) {
+ if (!pd->cd)
+ reason = "null digest?!";
+- else if (fetch->mask_offset != (int)pd->cd->mask_size)
++ else if (fetch->mask_offset != pd->cd->mask_size)
+ reason = "premature end of digest?!";
+ else if (!peerDigestUseful(pd))
+ reason = "useless digest";
+
+=== modified file 'src/store_digest.cc'
+--- src/store_digest.cc 2016-01-01 00:14:27 +0000
++++ src/store_digest.cc 2016-07-23 07:16:20 +0000
+@@ -76,36 +76,63 @@
+ static void storeDigestRewriteFinish(StoreEntry * e);
+ static EVH storeDigestSwapOutStep;
+ static void storeDigestCBlockSwapOut(StoreEntry * e);
+-static int storeDigestCalcCap(void);
+-static int storeDigestResize(void);
+ static void storeDigestAdd(const StoreEntry *);
+
++/// calculates digest capacity
++static uint64_t
++storeDigestCalcCap()
++{
++ /*
++ * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
++ * the bits are off). However, we do not have a formula to calculate the
++ * number of _entries_ we want to pre-allocate for.
++ */
++ const uint64_t hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
++ const uint64_t lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
++ const uint64_t e_count = StoreEntry::inUseCount();
++ uint64_t cap = e_count ? e_count : hi_cap;
++ debugs(71, 2, "have: " << e_count << ", want " << cap <<
++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
++
++ if (cap < lo_cap)
++ cap = lo_cap;
++
++ /* do not enforce hi_cap limit, average-based estimation may be wrong
++ *if (cap > hi_cap)
++ * cap = hi_cap;
++ */
++
++ // Bug 4534: we still have to set an upper-limit at some reasonable value though.
++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
++ const uint64_t absolute_max = (INT_MAX -8) / Config.digest.bits_per_entry;
++ if (cap > absolute_max) {
++ static time_t last_loud = 0;
++ if (last_loud < squid_curtime - 86400) {
++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
++ last_loud = squid_curtime;
++ } else {
++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
++ }
++ cap = absolute_max;
++ }
++
++ return cap;
++}
+ #endif /* USE_CACHE_DIGESTS */
+
+-static void
+-storeDigestRegisterWithCacheManager(void)
++void
++storeDigestInit(void)
+ {
+ Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1);
+-}
+-
+-/*
+- * PUBLIC FUNCTIONS
+- */
+-
+-void
+-storeDigestInit(void)
+-{
+- storeDigestRegisterWithCacheManager();
+
+ #if USE_CACHE_DIGESTS
+- const int cap = storeDigestCalcCap();
+-
+ if (!Config.onoff.digest_generation) {
+ store_digest = NULL;
+ debugs(71, 3, "Local cache digest generation disabled");
+ return;
+ }
+
++ const uint64_t cap = storeDigestCalcCap();
+ store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
+ debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " <<
+ (int) Config.digest.rebuild_period << "/" <<
+@@ -290,6 +317,31 @@
+ storeDigestRebuildResume();
+ }
+
++/// \returns true if we actually resized the digest
++static bool
++storeDigestResize()
++{
++ const uint64_t cap = storeDigestCalcCap();
++ assert(store_digest);
++ uint64_t diff;
++ if (cap > store_digest->capacity)
++ diff = cap - store_digest->capacity;
++ else
++ diff = store_digest->capacity - cap;
++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
++ diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
++ /* avoid minor adjustments */
++
++ if (diff <= store_digest->capacity / 10) {
++ debugs(71, 2, "small change, will not resize.");
++ return false;
++ } else {
++ debugs(71, 2, "big change, resizing.");
++ cacheDigestChangeCap(store_digest, cap);
++ }
++ return true;
++}
++
+ /* called be Rewrite to push Rebuild forward */
+ static void
+ storeDigestRebuildResume(void)
+@@ -439,7 +491,7 @@
+ assert(e);
+ /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */
+
+- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
++ if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) > store_digest->mask_size)
+ chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
+
+ e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
+@@ -451,7 +503,7 @@
+ sd_state.rewrite_offset += chunk_size;
+
+ /* are we done ? */
+- if (sd_state.rewrite_offset >= store_digest->mask_size)
++ if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest->mask_size)
+ storeDigestRewriteFinish(e);
+ else
+ eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false);
+@@ -467,60 +519,10 @@
+ sd_state.cblock.count = htonl(store_digest->count);
+ sd_state.cblock.del_count = htonl(store_digest->del_count);
+ sd_state.cblock.mask_size = htonl(store_digest->mask_size);
+- sd_state.cblock.bits_per_entry = (unsigned char)
+- Config.digest.bits_per_entry;
++ sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
+ sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount;
+ e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
+ }
+
+-/* calculates digest capacity */
+-static int
+-storeDigestCalcCap(void)
+-{
+- /*
+- * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
+- * the bits are off). However, we do not have a formula to calculate the
+- * number of _entries_ we want to pre-allocate for.
+- */
+- const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
+- const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
+- const int e_count = StoreEntry::inUseCount();
+- int cap = e_count ? e_count :hi_cap;
+- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap <<
+- " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
+-
+- if (cap < lo_cap)
+- cap = lo_cap;
+-
+- /* do not enforce hi_cap limit, average-based estimation may be wrong
+- *if (cap > hi_cap)
+- * cap = hi_cap;
+- */
+- return cap;
+-}
+-
+-/* returns true if we actually resized the digest */
+-static int
+-storeDigestResize(void)
+-{
+- const int cap = storeDigestCalcCap();
+- int diff;
+- assert(store_digest);
+- diff = abs(cap - store_digest->capacity);
+- debugs(71, 2, "storeDigestResize: " <<
+- store_digest->capacity << " -> " << cap << "; change: " <<
+- diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
+- /* avoid minor adjustments */
+-
+- if (diff <= store_digest->capacity / 10) {
+- debugs(71, 2, "storeDigestResize: small change, will not resize.");
+- return 0;
+- } else {
+- debugs(71, 2, "storeDigestResize: big change, resizing.");
+- cacheDigestChangeCap(store_digest, cap);
+- return 1;
+- }
+-}
+-
+ #endif /* USE_CACHE_DIGESTS */
+
+
+=== modified file 'src/tests/stub_CacheDigest.cc'
+--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
+@@ -16,11 +16,11 @@
+ class CacheDigestGuessStats;
+ class StoreEntry;
+
+-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
+ void cacheDigestDestroy(CacheDigest *) STUB
+ CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
+ void cacheDigestClear(CacheDigest * ) STUB
+-void cacheDigestChangeCap(CacheDigest *,int) STUB
++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
+ int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
+ void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
+ void cacheDigestDel(CacheDigest *, const cache_key *) STUB
+@@ -28,5 +28,4 @@
+ void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
+ void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
+-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
+-
++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
+
diff --git a/src/patches/squid/squid-3.5-14068.patch b/src/patches/squid/squid-3.5-14068.patch
new file mode 100644
index 0000000..4766e00
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14068.patch
@@ -0,0 +1,35 @@
+------------------------------------------------------------
+revno: 14068
+revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
+author: Anonymous <bigparrot(a)pirateperfection.com>
+committer: Amos Jeffries <squid3(a)treenet.co.nz>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 19:19:30 +1200
+message:
+ Bug #4542: authentication credentials IP TTL updated incorrectly
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
+# timestamp: 2016-07-23 07:24:05 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
+# 1wzqpbyi1rk5w6vg
+#
+# Begin patch
+=== modified file 'src/auth/User.cc'
+--- src/auth/User.cc 2016-01-01 00:14:27 +0000
++++ src/auth/User.cc 2016-07-23 07:19:30 +0000
+@@ -284,7 +284,7 @@
+ /* This ip has already been seen. */
+ found = 1;
+ /* update IP ttl */
+- ipdata->ip_expiretime = squid_curtime;
++ ipdata->ip_expiretime = squid_curtime + ::Config.authenticateIpTTL;
+ } else if (ipdata->ip_expiretime <= squid_curtime) {
+ /* This IP has expired - remove from the seen list */
+ dlinkDelete(&ipdata->node, &ip_list);
+
diff --git a/src/patches/squid/squid-3.5-14069.patch b/src/patches/squid/squid-3.5-14069.patch
new file mode 100644
index 0000000..15ca37a
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14069.patch
@@ -0,0 +1,30 @@
+------------------------------------------------------------
+revno: 14069
+revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
+parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+committer: Source Maintenance <squidadm(a)squid-cache.org>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 12:13:51 +0000
+message:
+ SourceFormat Enforcement
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squidadm(a)squid-cache.org-20160723121351-\
+# iuc8hwstrqd0l1dv
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
+# timestamp: 2016-07-23 12:50:56 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
+# cemledcltg8pkc28
+#
+# Begin patch
+=== modified file 'src/tests/stub_CacheDigest.cc'
+--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000
+@@ -29,3 +29,4 @@
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
+ void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
+ uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
++
+
--
2.9.2
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-07-24 11:44 [PATCH] squid 3.5.20: latest patches from upstream Matthias Fischer
@ 2016-08-02 15:14 ` Michael Tremer
2016-08-02 18:49 ` Matthias Fischer
0 siblings, 1 reply; 7+ messages in thread
From: Michael Tremer @ 2016-08-02 15:14 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 22346 bytes --]
This one won't apply onto next... :(
Could you please check why?
On Sun, 2016-07-24 at 13:44 +0200, Matthias Fischer wrote:
> Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
> ---
> lfs/squid | 3 +
> src/patches/squid/squid-3.5-14067.patch | 381
> ++++++++++++++++++++++++++++++++
> src/patches/squid/squid-3.5-14068.patch | 35 +++
> src/patches/squid/squid-3.5-14069.patch | 30 +++
> 4 files changed, 449 insertions(+)
> create mode 100644 src/patches/squid/squid-3.5-14067.patch
> create mode 100644 src/patches/squid/squid-3.5-14068.patch
> create mode 100644 src/patches/squid/squid-3.5-14069.patch
>
> diff --git a/lfs/squid b/lfs/squid
> index 5bc976e..a88f0c5 100644
> --- a/lfs/squid
> +++ b/lfs/squid
> @@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
> $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
> @$(PREBUILD)
> @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE)
> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
> 3.5-14067.patch
> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
> 3.5-14068.patch
> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
> 3.5-14069.patch
> cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-3.5.17-
> fix-max-file-descriptors.patch
>
> cd $(DIR_APP) && autoreconf -vfi
> diff --git a/src/patches/squid/squid-3.5-14067.patch
> b/src/patches/squid/squid-3.5-14067.patch
> new file mode 100644
> index 0000000..8d9cb21
> --- /dev/null
> +++ b/src/patches/squid/squid-3.5-14067.patch
> @@ -0,0 +1,381 @@
> +------------------------------------------------------------
> +revno: 14067
> +revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> +parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
> +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
> +committer: Amos Jeffries <squid3(a)treenet.co.nz>
> +branch nick: 3.5
> +timestamp: Sat 2016-07-23 19:16:20 +1200
> +message:
> + Bug 4534: assertion failure in xcalloc when using many cache_dir
> +------------------------------------------------------------
> +# Bazaar merge directive format 2 (Bazaar 0.90)
> +# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
> +# timestamp: 2016-07-23 07:24:01 +0000
> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
> +# vpjak1pq4uecadd2
> +#
> +# Begin patch
> +=== modified file 'src/CacheDigest.cc'
> +--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
> ++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
> +@@ -35,12 +35,12 @@
> + static uint32_t hashed_keys[4];
> +
> + static void
> +-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
> ++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
> + {
> +- const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
> ++ const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
> + assert(cd);
> + assert(capacity > 0 && bpe > 0);
> +- assert(mask_size > 0);
> ++ assert(mask_size != 0);
> + cd->capacity = capacity;
> + cd->bits_per_entry = bpe;
> + cd->mask_size = mask_size;
> +@@ -50,7 +50,7 @@
> + }
> +
> + CacheDigest *
> +-cacheDigestCreate(int capacity, int bpe)
> ++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
> + {
> + CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
> + assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely on 16
> byte keys */
> +@@ -97,7 +97,7 @@
> +
> + /* changes mask size, resets bits to 0, preserves "cd" pointer */
> + void
> +-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
> ++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
> + {
> + assert(cd);
> + cacheDigestClean(cd);
> +@@ -278,12 +278,12 @@
> + storeAppendPrintf(e, "%s digest: size: %d bytes\n",
> + label ? label : "", stats.bit_count / 8
> + );
> +- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
> ++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64
> " util: %d%%\n",
> + cd->count,
> + cd->capacity,
> + xpercentInt(cd->count, cd->capacity)
> + );
> +- storeAppendPrintf(e, "\t deletion attempts: %d\n",
> ++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
> + cd->del_count
> + );
> + storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util:
> %d%%\n",
> +@@ -297,16 +297,18 @@
> + );
> + }
> +
> +-size_t
> +-cacheDigestCalcMaskSize(int cap, int bpe)
> ++uint32_t
> ++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
> + {
> +- return (size_t) (cap * bpe + 7) / 8;
> ++ uint64_t bitCount = (cap * bpe) + 7;
> ++ assert(bitCount < INT_MAX); // dont 31-bit overflow later
> ++ return static_cast<uint32_t>(bitCount / 8);
> + }
> +
> + static void
> + cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
> + {
> +- const unsigned int bit_count = cd->mask_size * 8;
> ++ const uint32_t bit_count = cd->mask_size * 8;
> + unsigned int tmp_keys[4];
> + /* we must memcpy to ensure alignment */
> + memcpy(tmp_keys, key, sizeof(tmp_keys));
> +
> +=== modified file 'src/CacheDigest.h'
> +--- src/CacheDigest.h 2016-01-01 00:14:27 +0000
> ++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
> +@@ -22,23 +22,23 @@
> + {
> + public:
> + /* public, read-only */
> +- char *mask; /* bit mask */
> +- int mask_size; /* mask size in bytes */
> +- int capacity; /* expected maximum for .count, not a hard limit */
> +- int bits_per_entry; /* number of bits allocated for each entry from
> capacity */
> +- int count; /* number of digested entries */
> +- int del_count; /* number of deletions performed so far */
> ++ uint64_t count; /* number of digested entries */
> ++ uint64_t del_count; /* number of deletions performed so far */
> ++ uint64_t capacity; /* expected maximum for .count, not a hard
> limit */
> ++ char *mask; /* bit mask */
> ++ uint32_t mask_size; /* mask size in bytes */
> ++ int8_t bits_per_entry; /* number of bits allocated for each entry from
> capacity */
> + };
> +
> +-CacheDigest *cacheDigestCreate(int capacity, int bpe);
> ++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
> + void cacheDigestDestroy(CacheDigest * cd);
> + CacheDigest *cacheDigestClone(const CacheDigest * cd);
> + void cacheDigestClear(CacheDigest * cd);
> +-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
> ++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
> + int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
> + void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
> + void cacheDigestDel(CacheDigest * cd, const cache_key * key);
> +-size_t cacheDigestCalcMaskSize(int cap, int bpe);
> ++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
> + int cacheDigestBitUtil(const CacheDigest * cd);
> + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int
> real_hit, int guess_hit);
> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats,
> StoreEntry * sentry, const char *label);
> +
> +=== modified file 'src/PeerDigest.h'
> +--- src/PeerDigest.h 2016-01-01 00:14:27 +0000
> ++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
> +@@ -52,7 +52,7 @@
> + store_client *old_sc;
> + HttpRequest *request;
> + int offset;
> +- int mask_offset;
> ++ uint32_t mask_offset;
> + time_t start_time;
> + time_t resp_time;
> + time_t expires;
> +
> +=== modified file 'src/peer_digest.cc'
> +--- src/peer_digest.cc 2016-01-01 00:14:27 +0000
> ++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
> +@@ -754,7 +754,7 @@
> + if (!reason && !size) {
> + if (!pd->cd)
> + reason = "null digest?!";
> +- else if (fetch->mask_offset != (int)pd->cd->mask_size)
> ++ else if (fetch->mask_offset != pd->cd->mask_size)
> + reason = "premature end of digest?!";
> + else if (!peerDigestUseful(pd))
> + reason = "useless digest";
> +
> +=== modified file 'src/store_digest.cc'
> +--- src/store_digest.cc 2016-01-01 00:14:27 +0000
> ++++ src/store_digest.cc 2016-07-23 07:16:20 +0000
> +@@ -76,36 +76,63 @@
> + static void storeDigestRewriteFinish(StoreEntry * e);
> + static EVH storeDigestSwapOutStep;
> + static void storeDigestCBlockSwapOut(StoreEntry * e);
> +-static int storeDigestCalcCap(void);
> +-static int storeDigestResize(void);
> + static void storeDigestAdd(const StoreEntry *);
> +
> ++/// calculates digest capacity
> ++static uint64_t
> ++storeDigestCalcCap()
> ++{
> ++ /*
> ++ * To-Do: Bloom proved that the optimal filter utilization is 50% (half
> of
> ++ * the bits are off). However, we do not have a formula to calculate the
> ++ * number of _entries_ we want to pre-allocate for.
> ++ */
> ++ const uint64_t hi_cap = Store::Root().maxSize() /
> Config.Store.avgObjectSize;
> ++ const uint64_t lo_cap = 1 + Store::Root().currentSize() /
> Config.Store.avgObjectSize;
> ++ const uint64_t e_count = StoreEntry::inUseCount();
> ++ uint64_t cap = e_count ? e_count : hi_cap;
> ++ debugs(71, 2, "have: " << e_count << ", want " << cap <<
> ++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
> ++
> ++ if (cap < lo_cap)
> ++ cap = lo_cap;
> ++
> ++ /* do not enforce hi_cap limit, average-based estimation may be wrong
> ++ *if (cap > hi_cap)
> ++ * cap = hi_cap;
> ++ */
> ++
> ++ // Bug 4534: we still have to set an upper-limit at some reasonable
> value though.
> ++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
> ++ const uint64_t absolute_max = (INT_MAX -8) /
> Config.digest.bits_per_entry;
> ++ if (cap > absolute_max) {
> ++ static time_t last_loud = 0;
> ++ if (last_loud < squid_curtime - 86400) {
> ++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store "
> << cap << " entries. Limiting to " << absolute_max);
> ++ last_loud = squid_curtime;
> ++ } else {
> ++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << "
> entries. Limiting to " << absolute_max);
> ++ }
> ++ cap = absolute_max;
> ++ }
> ++
> ++ return cap;
> ++}
> + #endif /* USE_CACHE_DIGESTS */
> +
> +-static void
> +-storeDigestRegisterWithCacheManager(void)
> ++void
> ++storeDigestInit(void)
> + {
> + Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport,
> 0, 1);
> +-}
> +-
> +-/*
> +- * PUBLIC FUNCTIONS
> +- */
> +-
> +-void
> +-storeDigestInit(void)
> +-{
> +- storeDigestRegisterWithCacheManager();
> +
> + #if USE_CACHE_DIGESTS
> +- const int cap = storeDigestCalcCap();
> +-
> + if (!Config.onoff.digest_generation) {
> + store_digest = NULL;
> + debugs(71, 3, "Local cache digest generation disabled");
> + return;
> + }
> +
> ++ const uint64_t cap = storeDigestCalcCap();
> + store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
> + debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite
> every " <<
> + (int) Config.digest.rebuild_period << "/" <<
> +@@ -290,6 +317,31 @@
> + storeDigestRebuildResume();
> + }
> +
> ++/// \returns true if we actually resized the digest
> ++static bool
> ++storeDigestResize()
> ++{
> ++ const uint64_t cap = storeDigestCalcCap();
> ++ assert(store_digest);
> ++ uint64_t diff;
> ++ if (cap > store_digest->capacity)
> ++ diff = cap - store_digest->capacity;
> ++ else
> ++ diff = store_digest->capacity - cap;
> ++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
> ++ diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)"
> );
> ++ /* avoid minor adjustments */
> ++
> ++ if (diff <= store_digest->capacity / 10) {
> ++ debugs(71, 2, "small change, will not resize.");
> ++ return false;
> ++ } else {
> ++ debugs(71, 2, "big change, resizing.");
> ++ cacheDigestChangeCap(store_digest, cap);
> ++ }
> ++ return true;
> ++}
> ++
> + /* called be Rewrite to push Rebuild forward */
> + static void
> + storeDigestRebuildResume(void)
> +@@ -439,7 +491,7 @@
> + assert(e);
> + /* _add_ check that nothing bad happened while we were waiting @?@ @?@
> */
> +
> +- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
> ++ if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) >
> store_digest->mask_size)
> + chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
> +
> + e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
> +@@ -451,7 +503,7 @@
> + sd_state.rewrite_offset += chunk_size;
> +
> + /* are we done ? */
> +- if (sd_state.rewrite_offset >= store_digest->mask_size)
> ++ if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest-
> >mask_size)
> + storeDigestRewriteFinish(e);
> + else
> + eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data,
> 0.0, 1, false);
> +@@ -467,60 +519,10 @@
> + sd_state.cblock.count = htonl(store_digest->count);
> + sd_state.cblock.del_count = htonl(store_digest->del_count);
> + sd_state.cblock.mask_size = htonl(store_digest->mask_size);
> +- sd_state.cblock.bits_per_entry = (unsigned char)
> +- Config.digest.bits_per_entry;
> ++ sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
> + sd_state.cblock.hash_func_count = (unsigned char)
> CacheDigestHashFuncCount;
> + e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
> + }
> +
> +-/* calculates digest capacity */
> +-static int
> +-storeDigestCalcCap(void)
> +-{
> +- /*
> +- * To-Do: Bloom proved that the optimal filter utilization is 50% (half
> of
> +- * the bits are off). However, we do not have a formula to calculate the
> +- * number of _entries_ we want to pre-allocate for.
> +- */
> +- const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
> +- const int lo_cap = 1 + Store::Root().currentSize() /
> Config.Store.avgObjectSize;
> +- const int e_count = StoreEntry::inUseCount();
> +- int cap = e_count ? e_count :hi_cap;
> +- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " <<
> cap <<
> +- " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
> +-
> +- if (cap < lo_cap)
> +- cap = lo_cap;
> +-
> +- /* do not enforce hi_cap limit, average-based estimation may be wrong
> +- *if (cap > hi_cap)
> +- * cap = hi_cap;
> +- */
> +- return cap;
> +-}
> +-
> +-/* returns true if we actually resized the digest */
> +-static int
> +-storeDigestResize(void)
> +-{
> +- const int cap = storeDigestCalcCap();
> +- int diff;
> +- assert(store_digest);
> +- diff = abs(cap - store_digest->capacity);
> +- debugs(71, 2, "storeDigestResize: " <<
> +- store_digest->capacity << " -> " << cap << "; change: " <<
> +- diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)"
> );
> +- /* avoid minor adjustments */
> +-
> +- if (diff <= store_digest->capacity / 10) {
> +- debugs(71, 2, "storeDigestResize: small change, will not resize.");
> +- return 0;
> +- } else {
> +- debugs(71, 2, "storeDigestResize: big change, resizing.");
> +- cacheDigestChangeCap(store_digest, cap);
> +- return 1;
> +- }
> +-}
> +-
> + #endif /* USE_CACHE_DIGESTS */
> +
> +
> +=== modified file 'src/tests/stub_CacheDigest.cc'
> +--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
> ++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
> +@@ -16,11 +16,11 @@
> + class CacheDigestGuessStats;
> + class StoreEntry;
> +
> +-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
> ++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
> + void cacheDigestDestroy(CacheDigest *) STUB
> + CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
> + void cacheDigestClear(CacheDigest * ) STUB
> +-void cacheDigestChangeCap(CacheDigest *,int) STUB
> ++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
> + int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
> + void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
> + void cacheDigestDel(CacheDigest *, const cache_key *) STUB
> +@@ -28,5 +28,4 @@
> + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry
> *, const char *) STUB
> + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
> +-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
> +-
> ++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
> +
> diff --git a/src/patches/squid/squid-3.5-14068.patch
> b/src/patches/squid/squid-3.5-14068.patch
> new file mode 100644
> index 0000000..4766e00
> --- /dev/null
> +++ b/src/patches/squid/squid-3.5-14068.patch
> @@ -0,0 +1,35 @@
> +------------------------------------------------------------
> +revno: 14068
> +revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> +parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
> +author: Anonymous <bigparrot(a)pirateperfection.com>
> +committer: Amos Jeffries <squid3(a)treenet.co.nz>
> +branch nick: 3.5
> +timestamp: Sat 2016-07-23 19:19:30 +1200
> +message:
> + Bug #4542: authentication credentials IP TTL updated incorrectly
> +------------------------------------------------------------
> +# Bazaar merge directive format 2 (Bazaar 0.90)
> +# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
> +# timestamp: 2016-07-23 07:24:05 +0000
> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
> +# 1wzqpbyi1rk5w6vg
> +#
> +# Begin patch
> +=== modified file 'src/auth/User.cc'
> +--- src/auth/User.cc 2016-01-01 00:14:27 +0000
> ++++ src/auth/User.cc 2016-07-23 07:19:30 +0000
> +@@ -284,7 +284,7 @@
> + /* This ip has already been seen. */
> + found = 1;
> + /* update IP ttl */
> +- ipdata->ip_expiretime = squid_curtime;
> ++ ipdata->ip_expiretime = squid_curtime +
> ::Config.authenticateIpTTL;
> + } else if (ipdata->ip_expiretime <= squid_curtime) {
> + /* This IP has expired - remove from the seen list */
> + dlinkDelete(&ipdata->node, &ip_list);
> +
> diff --git a/src/patches/squid/squid-3.5-14069.patch
> b/src/patches/squid/squid-3.5-14069.patch
> new file mode 100644
> index 0000000..15ca37a
> --- /dev/null
> +++ b/src/patches/squid/squid-3.5-14069.patch
> @@ -0,0 +1,30 @@
> +------------------------------------------------------------
> +revno: 14069
> +revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
> +parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> +committer: Source Maintenance <squidadm(a)squid-cache.org>
> +branch nick: 3.5
> +timestamp: Sat 2016-07-23 12:13:51 +0000
> +message:
> + SourceFormat Enforcement
> +------------------------------------------------------------
> +# Bazaar merge directive format 2 (Bazaar 0.90)
> +# revision_id: squidadm(a)squid-cache.org-20160723121351-\
> +# iuc8hwstrqd0l1dv
> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
> +# timestamp: 2016-07-23 12:50:56 +0000
> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> +# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
> +# cemledcltg8pkc28
> +#
> +# Begin patch
> +=== modified file 'src/tests/stub_CacheDigest.cc'
> +--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
> ++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000
> +@@ -29,3 +29,4 @@
> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry
> *, const char *) STUB
> + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
> + uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
> ++
> +
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-08-02 15:14 ` Michael Tremer
@ 2016-08-02 18:49 ` Matthias Fischer
2016-08-03 10:47 ` Michael Tremer
0 siblings, 1 reply; 7+ messages in thread
From: Matthias Fischer @ 2016-08-02 18:49 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 22116 bytes --]
Hi,
On 02.08.2016 17:14, Michael Tremer wrote:
> This one won't apply onto next... :(
>
> Could you please check why?
Its always the same here (*sigh*): sources seemed to be missing.
Please check again. Its running here.
Best,
Matthias
> On Sun, 2016-07-24 at 13:44 +0200, Matthias Fischer wrote:
>> Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
>> ---
>> lfs/squid | 3 +
>> src/patches/squid/squid-3.5-14067.patch | 381
>> ++++++++++++++++++++++++++++++++
>> src/patches/squid/squid-3.5-14068.patch | 35 +++
>> src/patches/squid/squid-3.5-14069.patch | 30 +++
>> 4 files changed, 449 insertions(+)
>> create mode 100644 src/patches/squid/squid-3.5-14067.patch
>> create mode 100644 src/patches/squid/squid-3.5-14068.patch
>> create mode 100644 src/patches/squid/squid-3.5-14069.patch
>>
>> diff --git a/lfs/squid b/lfs/squid
>> index 5bc976e..a88f0c5 100644
>> --- a/lfs/squid
>> +++ b/lfs/squid
>> @@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
>> $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
>> @$(PREBUILD)
>> @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE)
>> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
>> 3.5-14067.patch
>> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
>> 3.5-14068.patch
>> + cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-
>> 3.5-14069.patch
>> cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-3.5.17-
>> fix-max-file-descriptors.patch
>>
>> cd $(DIR_APP) && autoreconf -vfi
>> diff --git a/src/patches/squid/squid-3.5-14067.patch
>> b/src/patches/squid/squid-3.5-14067.patch
>> new file mode 100644
>> index 0000000..8d9cb21
>> --- /dev/null
>> +++ b/src/patches/squid/squid-3.5-14067.patch
>> @@ -0,0 +1,381 @@
>> +------------------------------------------------------------
>> +revno: 14067
>> +revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> +parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
>> +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
>> +committer: Amos Jeffries <squid3(a)treenet.co.nz>
>> +branch nick: 3.5
>> +timestamp: Sat 2016-07-23 19:16:20 +1200
>> +message:
>> + Bug 4534: assertion failure in xcalloc when using many cache_dir
>> +------------------------------------------------------------
>> +# Bazaar merge directive format 2 (Bazaar 0.90)
>> +# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
>> +# timestamp: 2016-07-23 07:24:01 +0000
>> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
>> +# vpjak1pq4uecadd2
>> +#
>> +# Begin patch
>> +=== modified file 'src/CacheDigest.cc'
>> +--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
>> ++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
>> +@@ -35,12 +35,12 @@
>> + static uint32_t hashed_keys[4];
>> +
>> + static void
>> +-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
>> ++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
>> + {
>> +- const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
>> ++ const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
>> + assert(cd);
>> + assert(capacity > 0 && bpe > 0);
>> +- assert(mask_size > 0);
>> ++ assert(mask_size != 0);
>> + cd->capacity = capacity;
>> + cd->bits_per_entry = bpe;
>> + cd->mask_size = mask_size;
>> +@@ -50,7 +50,7 @@
>> + }
>> +
>> + CacheDigest *
>> +-cacheDigestCreate(int capacity, int bpe)
>> ++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
>> + {
>> + CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
>> + assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely on 16
>> byte keys */
>> +@@ -97,7 +97,7 @@
>> +
>> + /* changes mask size, resets bits to 0, preserves "cd" pointer */
>> + void
>> +-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
>> ++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
>> + {
>> + assert(cd);
>> + cacheDigestClean(cd);
>> +@@ -278,12 +278,12 @@
>> + storeAppendPrintf(e, "%s digest: size: %d bytes\n",
>> + label ? label : "", stats.bit_count / 8
>> + );
>> +- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
>> ++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64
>> " util: %d%%\n",
>> + cd->count,
>> + cd->capacity,
>> + xpercentInt(cd->count, cd->capacity)
>> + );
>> +- storeAppendPrintf(e, "\t deletion attempts: %d\n",
>> ++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
>> + cd->del_count
>> + );
>> + storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util:
>> %d%%\n",
>> +@@ -297,16 +297,18 @@
>> + );
>> + }
>> +
>> +-size_t
>> +-cacheDigestCalcMaskSize(int cap, int bpe)
>> ++uint32_t
>> ++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
>> + {
>> +- return (size_t) (cap * bpe + 7) / 8;
>> ++ uint64_t bitCount = (cap * bpe) + 7;
>> ++ assert(bitCount < INT_MAX); // dont 31-bit overflow later
>> ++ return static_cast<uint32_t>(bitCount / 8);
>> + }
>> +
>> + static void
>> + cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
>> + {
>> +- const unsigned int bit_count = cd->mask_size * 8;
>> ++ const uint32_t bit_count = cd->mask_size * 8;
>> + unsigned int tmp_keys[4];
>> + /* we must memcpy to ensure alignment */
>> + memcpy(tmp_keys, key, sizeof(tmp_keys));
>> +
>> +=== modified file 'src/CacheDigest.h'
>> +--- src/CacheDigest.h 2016-01-01 00:14:27 +0000
>> ++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
>> +@@ -22,23 +22,23 @@
>> + {
>> + public:
>> + /* public, read-only */
>> +- char *mask; /* bit mask */
>> +- int mask_size; /* mask size in bytes */
>> +- int capacity; /* expected maximum for .count, not a hard limit */
>> +- int bits_per_entry; /* number of bits allocated for each entry from
>> capacity */
>> +- int count; /* number of digested entries */
>> +- int del_count; /* number of deletions performed so far */
>> ++ uint64_t count; /* number of digested entries */
>> ++ uint64_t del_count; /* number of deletions performed so far */
>> ++ uint64_t capacity; /* expected maximum for .count, not a hard
>> limit */
>> ++ char *mask; /* bit mask */
>> ++ uint32_t mask_size; /* mask size in bytes */
>> ++ int8_t bits_per_entry; /* number of bits allocated for each entry from
>> capacity */
>> + };
>> +
>> +-CacheDigest *cacheDigestCreate(int capacity, int bpe);
>> ++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
>> + void cacheDigestDestroy(CacheDigest * cd);
>> + CacheDigest *cacheDigestClone(const CacheDigest * cd);
>> + void cacheDigestClear(CacheDigest * cd);
>> +-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
>> ++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
>> + int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
>> + void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
>> + void cacheDigestDel(CacheDigest * cd, const cache_key * key);
>> +-size_t cacheDigestCalcMaskSize(int cap, int bpe);
>> ++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
>> + int cacheDigestBitUtil(const CacheDigest * cd);
>> + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int
>> real_hit, int guess_hit);
>> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats,
>> StoreEntry * sentry, const char *label);
>> +
>> +=== modified file 'src/PeerDigest.h'
>> +--- src/PeerDigest.h 2016-01-01 00:14:27 +0000
>> ++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
>> +@@ -52,7 +52,7 @@
>> + store_client *old_sc;
>> + HttpRequest *request;
>> + int offset;
>> +- int mask_offset;
>> ++ uint32_t mask_offset;
>> + time_t start_time;
>> + time_t resp_time;
>> + time_t expires;
>> +
>> +=== modified file 'src/peer_digest.cc'
>> +--- src/peer_digest.cc 2016-01-01 00:14:27 +0000
>> ++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
>> +@@ -754,7 +754,7 @@
>> + if (!reason && !size) {
>> + if (!pd->cd)
>> + reason = "null digest?!";
>> +- else if (fetch->mask_offset != (int)pd->cd->mask_size)
>> ++ else if (fetch->mask_offset != pd->cd->mask_size)
>> + reason = "premature end of digest?!";
>> + else if (!peerDigestUseful(pd))
>> + reason = "useless digest";
>> +
>> +=== modified file 'src/store_digest.cc'
>> +--- src/store_digest.cc 2016-01-01 00:14:27 +0000
>> ++++ src/store_digest.cc 2016-07-23 07:16:20 +0000
>> +@@ -76,36 +76,63 @@
>> + static void storeDigestRewriteFinish(StoreEntry * e);
>> + static EVH storeDigestSwapOutStep;
>> + static void storeDigestCBlockSwapOut(StoreEntry * e);
>> +-static int storeDigestCalcCap(void);
>> +-static int storeDigestResize(void);
>> + static void storeDigestAdd(const StoreEntry *);
>> +
>> ++/// calculates digest capacity
>> ++static uint64_t
>> ++storeDigestCalcCap()
>> ++{
>> ++ /*
>> ++ * To-Do: Bloom proved that the optimal filter utilization is 50% (half
>> of
>> ++ * the bits are off). However, we do not have a formula to calculate the
>> ++ * number of _entries_ we want to pre-allocate for.
>> ++ */
>> ++ const uint64_t hi_cap = Store::Root().maxSize() /
>> Config.Store.avgObjectSize;
>> ++ const uint64_t lo_cap = 1 + Store::Root().currentSize() /
>> Config.Store.avgObjectSize;
>> ++ const uint64_t e_count = StoreEntry::inUseCount();
>> ++ uint64_t cap = e_count ? e_count : hi_cap;
>> ++ debugs(71, 2, "have: " << e_count << ", want " << cap <<
>> ++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
>> ++
>> ++ if (cap < lo_cap)
>> ++ cap = lo_cap;
>> ++
>> ++ /* do not enforce hi_cap limit, average-based estimation may be wrong
>> ++ *if (cap > hi_cap)
>> ++ * cap = hi_cap;
>> ++ */
>> ++
>> ++ // Bug 4534: we still have to set an upper-limit at some reasonable
>> value though.
>> ++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
>> ++ const uint64_t absolute_max = (INT_MAX -8) /
>> Config.digest.bits_per_entry;
>> ++ if (cap > absolute_max) {
>> ++ static time_t last_loud = 0;
>> ++ if (last_loud < squid_curtime - 86400) {
>> ++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store "
>> << cap << " entries. Limiting to " << absolute_max);
>> ++ last_loud = squid_curtime;
>> ++ } else {
>> ++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << "
>> entries. Limiting to " << absolute_max);
>> ++ }
>> ++ cap = absolute_max;
>> ++ }
>> ++
>> ++ return cap;
>> ++}
>> + #endif /* USE_CACHE_DIGESTS */
>> +
>> +-static void
>> +-storeDigestRegisterWithCacheManager(void)
>> ++void
>> ++storeDigestInit(void)
>> + {
>> + Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport,
>> 0, 1);
>> +-}
>> +-
>> +-/*
>> +- * PUBLIC FUNCTIONS
>> +- */
>> +-
>> +-void
>> +-storeDigestInit(void)
>> +-{
>> +- storeDigestRegisterWithCacheManager();
>> +
>> + #if USE_CACHE_DIGESTS
>> +- const int cap = storeDigestCalcCap();
>> +-
>> + if (!Config.onoff.digest_generation) {
>> + store_digest = NULL;
>> + debugs(71, 3, "Local cache digest generation disabled");
>> + return;
>> + }
>> +
>> ++ const uint64_t cap = storeDigestCalcCap();
>> + store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
>> + debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite
>> every " <<
>> + (int) Config.digest.rebuild_period << "/" <<
>> +@@ -290,6 +317,31 @@
>> + storeDigestRebuildResume();
>> + }
>> +
>> ++/// \returns true if we actually resized the digest
>> ++static bool
>> ++storeDigestResize()
>> ++{
>> ++ const uint64_t cap = storeDigestCalcCap();
>> ++ assert(store_digest);
>> ++ uint64_t diff;
>> ++ if (cap > store_digest->capacity)
>> ++ diff = cap - store_digest->capacity;
>> ++ else
>> ++ diff = store_digest->capacity - cap;
>> ++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
>> ++ diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)"
>> );
>> ++ /* avoid minor adjustments */
>> ++
>> ++ if (diff <= store_digest->capacity / 10) {
>> ++ debugs(71, 2, "small change, will not resize.");
>> ++ return false;
>> ++ } else {
>> ++ debugs(71, 2, "big change, resizing.");
>> ++ cacheDigestChangeCap(store_digest, cap);
>> ++ }
>> ++ return true;
>> ++}
>> ++
>> + /* called be Rewrite to push Rebuild forward */
>> + static void
>> + storeDigestRebuildResume(void)
>> +@@ -439,7 +491,7 @@
>> + assert(e);
>> + /* _add_ check that nothing bad happened while we were waiting @?@ @?@
>> */
>> +
>> +- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
>> ++ if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) >
>> store_digest->mask_size)
>> + chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
>> +
>> + e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
>> +@@ -451,7 +503,7 @@
>> + sd_state.rewrite_offset += chunk_size;
>> +
>> + /* are we done ? */
>> +- if (sd_state.rewrite_offset >= store_digest->mask_size)
>> ++ if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest-
>> >mask_size)
>> + storeDigestRewriteFinish(e);
>> + else
>> + eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data,
>> 0.0, 1, false);
>> +@@ -467,60 +519,10 @@
>> + sd_state.cblock.count = htonl(store_digest->count);
>> + sd_state.cblock.del_count = htonl(store_digest->del_count);
>> + sd_state.cblock.mask_size = htonl(store_digest->mask_size);
>> +- sd_state.cblock.bits_per_entry = (unsigned char)
>> +- Config.digest.bits_per_entry;
>> ++ sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
>> + sd_state.cblock.hash_func_count = (unsigned char)
>> CacheDigestHashFuncCount;
>> + e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
>> + }
>> +
>> +-/* calculates digest capacity */
>> +-static int
>> +-storeDigestCalcCap(void)
>> +-{
>> +- /*
>> +- * To-Do: Bloom proved that the optimal filter utilization is 50% (half
>> of
>> +- * the bits are off). However, we do not have a formula to calculate the
>> +- * number of _entries_ we want to pre-allocate for.
>> +- */
>> +- const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
>> +- const int lo_cap = 1 + Store::Root().currentSize() /
>> Config.Store.avgObjectSize;
>> +- const int e_count = StoreEntry::inUseCount();
>> +- int cap = e_count ? e_count :hi_cap;
>> +- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " <<
>> cap <<
>> +- " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
>> +-
>> +- if (cap < lo_cap)
>> +- cap = lo_cap;
>> +-
>> +- /* do not enforce hi_cap limit, average-based estimation may be wrong
>> +- *if (cap > hi_cap)
>> +- * cap = hi_cap;
>> +- */
>> +- return cap;
>> +-}
>> +-
>> +-/* returns true if we actually resized the digest */
>> +-static int
>> +-storeDigestResize(void)
>> +-{
>> +- const int cap = storeDigestCalcCap();
>> +- int diff;
>> +- assert(store_digest);
>> +- diff = abs(cap - store_digest->capacity);
>> +- debugs(71, 2, "storeDigestResize: " <<
>> +- store_digest->capacity << " -> " << cap << "; change: " <<
>> +- diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)"
>> );
>> +- /* avoid minor adjustments */
>> +-
>> +- if (diff <= store_digest->capacity / 10) {
>> +- debugs(71, 2, "storeDigestResize: small change, will not resize.");
>> +- return 0;
>> +- } else {
>> +- debugs(71, 2, "storeDigestResize: big change, resizing.");
>> +- cacheDigestChangeCap(store_digest, cap);
>> +- return 1;
>> +- }
>> +-}
>> +-
>> + #endif /* USE_CACHE_DIGESTS */
>> +
>> +
>> +=== modified file 'src/tests/stub_CacheDigest.cc'
>> +--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
>> ++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
>> +@@ -16,11 +16,11 @@
>> + class CacheDigestGuessStats;
>> + class StoreEntry;
>> +
>> +-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
>> ++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
>> + void cacheDigestDestroy(CacheDigest *) STUB
>> + CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
>> + void cacheDigestClear(CacheDigest * ) STUB
>> +-void cacheDigestChangeCap(CacheDigest *,int) STUB
>> ++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
>> + int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
>> + void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
>> + void cacheDigestDel(CacheDigest *, const cache_key *) STUB
>> +@@ -28,5 +28,4 @@
>> + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
>> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry
>> *, const char *) STUB
>> + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
>> +-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
>> +-
>> ++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
>> +
>> diff --git a/src/patches/squid/squid-3.5-14068.patch
>> b/src/patches/squid/squid-3.5-14068.patch
>> new file mode 100644
>> index 0000000..4766e00
>> --- /dev/null
>> +++ b/src/patches/squid/squid-3.5-14068.patch
>> @@ -0,0 +1,35 @@
>> +------------------------------------------------------------
>> +revno: 14068
>> +revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> +parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
>> +author: Anonymous <bigparrot(a)pirateperfection.com>
>> +committer: Amos Jeffries <squid3(a)treenet.co.nz>
>> +branch nick: 3.5
>> +timestamp: Sat 2016-07-23 19:19:30 +1200
>> +message:
>> + Bug #4542: authentication credentials IP TTL updated incorrectly
>> +------------------------------------------------------------
>> +# Bazaar merge directive format 2 (Bazaar 0.90)
>> +# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
>> +# timestamp: 2016-07-23 07:24:05 +0000
>> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
>> +# 1wzqpbyi1rk5w6vg
>> +#
>> +# Begin patch
>> +=== modified file 'src/auth/User.cc'
>> +--- src/auth/User.cc 2016-01-01 00:14:27 +0000
>> ++++ src/auth/User.cc 2016-07-23 07:19:30 +0000
>> +@@ -284,7 +284,7 @@
>> + /* This ip has already been seen. */
>> + found = 1;
>> + /* update IP ttl */
>> +- ipdata->ip_expiretime = squid_curtime;
>> ++ ipdata->ip_expiretime = squid_curtime +
>> ::Config.authenticateIpTTL;
>> + } else if (ipdata->ip_expiretime <= squid_curtime) {
>> + /* This IP has expired - remove from the seen list */
>> + dlinkDelete(&ipdata->node, &ip_list);
>> +
>> diff --git a/src/patches/squid/squid-3.5-14069.patch
>> b/src/patches/squid/squid-3.5-14069.patch
>> new file mode 100644
>> index 0000000..15ca37a
>> --- /dev/null
>> +++ b/src/patches/squid/squid-3.5-14069.patch
>> @@ -0,0 +1,30 @@
>> +------------------------------------------------------------
>> +revno: 14069
>> +revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
>> +parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> +committer: Source Maintenance <squidadm(a)squid-cache.org>
>> +branch nick: 3.5
>> +timestamp: Sat 2016-07-23 12:13:51 +0000
>> +message:
>> + SourceFormat Enforcement
>> +------------------------------------------------------------
>> +# Bazaar merge directive format 2 (Bazaar 0.90)
>> +# revision_id: squidadm(a)squid-cache.org-20160723121351-\
>> +# iuc8hwstrqd0l1dv
>> +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
>> +# timestamp: 2016-07-23 12:50:56 +0000
>> +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> +# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
>> +# cemledcltg8pkc28
>> +#
>> +# Begin patch
>> +=== modified file 'src/tests/stub_CacheDigest.cc'
>> +--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
>> ++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000
>> +@@ -29,3 +29,4 @@
>> + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry
>> *, const char *) STUB
>> + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
>> + uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
>> ++
>> +
>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-08-02 18:49 ` Matthias Fischer
@ 2016-08-03 10:47 ` Michael Tremer
2016-08-03 18:49 ` Matthias Fischer
2016-08-18 16:07 ` Matthias Fischer
0 siblings, 2 replies; 7+ messages in thread
From: Michael Tremer @ 2016-08-03 10:47 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 26509 bytes --]
Hi,
I didn't check the source, yet because it fails to apply before that:
[root(a)rice-oxley ipfire-2.x]# pwclient git-am -s 692
Applying patch #692 using 'git am -s'
Description: squid 3.5.20: latest patches from upstream
Applying: squid 3.5.20: latest patches from upstream
.git/rebase-apply/patch:50: trailing whitespace.
#
.git/rebase-apply/patch:57: trailing whitespace.
.git/rebase-apply/patch:73: trailing whitespace.
.git/rebase-apply/patch:81: trailing whitespace.
.git/rebase-apply/patch:107: trailing whitespace.
error: patch failed: lfs/squid:70
error: lfs/squid: patch does not apply
Patch failed at 0001 squid 3.5.20: latest patches from upstream
The copy of the patch that failed is found in: .git/rebase-apply/patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
'git am' failed with exit status 128
On Tue, 2016-08-02 at 20:49 +0200, Matthias Fischer wrote:
> Hi,
>
> On 02.08.2016 17:14, Michael Tremer wrote:
> > This one won't apply onto next... :(
> >
> > Could you please check why?
>
> Its always the same here (*sigh*): sources seemed to be missing.
>
> Please check again. Its running here.
>
> Best,
> Matthias
>
> > On Sun, 2016-07-24 at 13:44 +0200, Matthias Fischer wrote:
> > > Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
> > > ---
> > > lfs/squid | 3 +
> > > src/patches/squid/squid-3.5-14067.patch | 381
> > > ++++++++++++++++++++++++++++++++
> > > src/patches/squid/squid-3.5-14068.patch | 35 +++
> > > src/patches/squid/squid-3.5-14069.patch | 30 +++
> > > 4 files changed, 449 insertions(+)
> > > create mode 100644 src/patches/squid/squid-3.5-14067.patch
> > > create mode 100644 src/patches/squid/squid-3.5-14068.patch
> > > create mode 100644 src/patches/squid/squid-3.5-14069.patch
> > >
> > > diff --git a/lfs/squid b/lfs/squid
> > > index 5bc976e..a88f0c5 100644
> > > --- a/lfs/squid
> > > +++ b/lfs/squid
> > > @@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
> > > $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
> > > @$(PREBUILD)
> > > @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf
> > > $(DIR_DL)/$(DL_FILE)
> > > + cd $(DIR_APP) && patch -Np0 -i
> > > $(DIR_SRC)/src/patches/squid/squid-
> > > 3.5-14067.patch
> > > + cd $(DIR_APP) && patch -Np0 -i
> > > $(DIR_SRC)/src/patches/squid/squid-
> > > 3.5-14068.patch
> > > + cd $(DIR_APP) && patch -Np0 -i
> > > $(DIR_SRC)/src/patches/squid/squid-
> > > 3.5-14069.patch
> > > cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-
> > > 3.5.17-
> > > fix-max-file-descriptors.patch
> > >
> > > cd $(DIR_APP) && autoreconf -vfi
> > > diff --git a/src/patches/squid/squid-3.5-14067.patch
> > > b/src/patches/squid/squid-3.5-14067.patch
> > > new file mode 100644
> > > index 0000000..8d9cb21
> > > --- /dev/null
> > > +++ b/src/patches/squid/squid-3.5-14067.patch
> > > @@ -0,0 +1,381 @@
> > > +------------------------------------------------------------
> > > +revno: 14067
> > > +revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> > > +parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
> > > +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
> > > +committer: Amos Jeffries <squid3(a)treenet.co.nz>
> > > +branch nick: 3.5
> > > +timestamp: Sat 2016-07-23 19:16:20 +1200
> > > +message:
> > > + Bug 4534: assertion failure in xcalloc when using many cache_dir
> > > +------------------------------------------------------------
> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
> > > +# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
> > > +# timestamp: 2016-07-23 07:24:01 +0000
> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
> > > +# vpjak1pq4uecadd2
> > > +#
> > > +# Begin patch
> > > +=== modified file 'src/CacheDigest.cc'
> > > +--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
> > > ++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
> > > +@@ -35,12 +35,12 @@
> > > + static uint32_t hashed_keys[4];
> > > +
> > > + static void
> > > +-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
> > > ++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
> > > + {
> > > +- const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
> > > ++ const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
> > > + assert(cd);
> > > + assert(capacity > 0 && bpe > 0);
> > > +- assert(mask_size > 0);
> > > ++ assert(mask_size != 0);
> > > + cd->capacity = capacity;
> > > + cd->bits_per_entry = bpe;
> > > + cd->mask_size = mask_size;
> > > +@@ -50,7 +50,7 @@
> > > + }
> > > +
> > > + CacheDigest *
> > > +-cacheDigestCreate(int capacity, int bpe)
> > > ++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
> > > + {
> > > + CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
> > > + assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely
> > > on 16
> > > byte keys */
> > > +@@ -97,7 +97,7 @@
> > > +
> > > + /* changes mask size, resets bits to 0, preserves "cd" pointer */
> > > + void
> > > +-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
> > > ++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
> > > + {
> > > + assert(cd);
> > > + cacheDigestClean(cd);
> > > +@@ -278,12 +278,12 @@
> > > + storeAppendPrintf(e, "%s digest: size: %d bytes\n",
> > > + label ? label : "", stats.bit_count / 8
> > > + );
> > > +- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util:
> > > %d%%\n",
> > > ++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %"
> > > PRIu64
> > > " util: %d%%\n",
> > > + cd->count,
> > > + cd->capacity,
> > > + xpercentInt(cd->count, cd->capacity)
> > > + );
> > > +- storeAppendPrintf(e, "\t deletion attempts: %d\n",
> > > ++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
> > > + cd->del_count
> > > + );
> > > + storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d
> > > util:
> > > %d%%\n",
> > > +@@ -297,16 +297,18 @@
> > > + );
> > > + }
> > > +
> > > +-size_t
> > > +-cacheDigestCalcMaskSize(int cap, int bpe)
> > > ++uint32_t
> > > ++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
> > > + {
> > > +- return (size_t) (cap * bpe + 7) / 8;
> > > ++ uint64_t bitCount = (cap * bpe) + 7;
> > > ++ assert(bitCount < INT_MAX); // dont 31-bit overflow later
> > > ++ return static_cast<uint32_t>(bitCount / 8);
> > > + }
> > > +
> > > + static void
> > > + cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
> > > + {
> > > +- const unsigned int bit_count = cd->mask_size * 8;
> > > ++ const uint32_t bit_count = cd->mask_size * 8;
> > > + unsigned int tmp_keys[4];
> > > + /* we must memcpy to ensure alignment */
> > > + memcpy(tmp_keys, key, sizeof(tmp_keys));
> > > +
> > > +=== modified file 'src/CacheDigest.h'
> > > +--- src/CacheDigest.h 2016-01-01 00:14:27 +0000
> > > ++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
> > > +@@ -22,23 +22,23 @@
> > > + {
> > > + public:
> > > + /* public, read-only */
> > > +- char *mask; /* bit mask */
> > > +- int mask_size; /* mask size in bytes */
> > > +- int capacity; /* expected maximum for .count, not a hard limit
> > > */
> > > +- int bits_per_entry; /* number of bits allocated for each entry
> > > from
> > > capacity */
> > > +- int count; /* number of digested entries */
> > > +- int del_count; /* number of deletions performed so far */
> > > ++ uint64_t count; /* number of digested entries */
> > > ++ uint64_t del_count; /* number of deletions performed so far */
> > > ++ uint64_t capacity; /* expected maximum for .count, not a hard
> > > limit */
> > > ++ char *mask; /* bit mask */
> > > ++ uint32_t mask_size; /* mask size in bytes */
> > > ++ int8_t bits_per_entry; /* number of bits allocated for each entry
> > > from
> > > capacity */
> > > + };
> > > +
> > > +-CacheDigest *cacheDigestCreate(int capacity, int bpe);
> > > ++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
> > > + void cacheDigestDestroy(CacheDigest * cd);
> > > + CacheDigest *cacheDigestClone(const CacheDigest * cd);
> > > + void cacheDigestClear(CacheDigest * cd);
> > > +-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
> > > ++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
> > > + int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
> > > + void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
> > > + void cacheDigestDel(CacheDigest * cd, const cache_key * key);
> > > +-size_t cacheDigestCalcMaskSize(int cap, int bpe);
> > > ++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
> > > + int cacheDigestBitUtil(const CacheDigest * cd);
> > > + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int
> > > real_hit, int guess_hit);
> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats,
> > > StoreEntry * sentry, const char *label);
> > > +
> > > +=== modified file 'src/PeerDigest.h'
> > > +--- src/PeerDigest.h 2016-01-01 00:14:27 +0000
> > > ++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
> > > +@@ -52,7 +52,7 @@
> > > + store_client *old_sc;
> > > + HttpRequest *request;
> > > + int offset;
> > > +- int mask_offset;
> > > ++ uint32_t mask_offset;
> > > + time_t start_time;
> > > + time_t resp_time;
> > > + time_t expires;
> > > +
> > > +=== modified file 'src/peer_digest.cc'
> > > +--- src/peer_digest.cc 2016-01-01 00:14:27 +0000
> > > ++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
> > > +@@ -754,7 +754,7 @@
> > > + if (!reason && !size) {
> > > + if (!pd->cd)
> > > + reason = "null digest?!";
> > > +- else if (fetch->mask_offset != (int)pd->cd->mask_size)
> > > ++ else if (fetch->mask_offset != pd->cd->mask_size)
> > > + reason = "premature end of digest?!";
> > > + else if (!peerDigestUseful(pd))
> > > + reason = "useless digest";
> > > +
> > > +=== modified file 'src/store_digest.cc'
> > > +--- src/store_digest.cc 2016-01-01 00:14:27 +0000
> > > ++++ src/store_digest.cc 2016-07-23 07:16:20 +0000
> > > +@@ -76,36 +76,63 @@
> > > + static void storeDigestRewriteFinish(StoreEntry * e);
> > > + static EVH storeDigestSwapOutStep;
> > > + static void storeDigestCBlockSwapOut(StoreEntry * e);
> > > +-static int storeDigestCalcCap(void);
> > > +-static int storeDigestResize(void);
> > > + static void storeDigestAdd(const StoreEntry *);
> > > +
> > > ++/// calculates digest capacity
> > > ++static uint64_t
> > > ++storeDigestCalcCap()
> > > ++{
> > > ++ /*
> > > ++ * To-Do: Bloom proved that the optimal filter utilization is 50%
> > > (half
> > > of
> > > ++ * the bits are off). However, we do not have a formula to calculate
> > > the
> > > ++ * number of _entries_ we want to pre-allocate for.
> > > ++ */
> > > ++ const uint64_t hi_cap = Store::Root().maxSize() /
> > > Config.Store.avgObjectSize;
> > > ++ const uint64_t lo_cap = 1 + Store::Root().currentSize() /
> > > Config.Store.avgObjectSize;
> > > ++ const uint64_t e_count = StoreEntry::inUseCount();
> > > ++ uint64_t cap = e_count ? e_count : hi_cap;
> > > ++ debugs(71, 2, "have: " << e_count << ", want " << cap <<
> > > ++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
> > > ++
> > > ++ if (cap < lo_cap)
> > > ++ cap = lo_cap;
> > > ++
> > > ++ /* do not enforce hi_cap limit, average-based estimation may be
> > > wrong
> > > ++ *if (cap > hi_cap)
> > > ++ * cap = hi_cap;
> > > ++ */
> > > ++
> > > ++ // Bug 4534: we still have to set an upper-limit at some reasonable
> > > value though.
> > > ++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
> > > ++ const uint64_t absolute_max = (INT_MAX -8) /
> > > Config.digest.bits_per_entry;
> > > ++ if (cap > absolute_max) {
> > > ++ static time_t last_loud = 0;
> > > ++ if (last_loud < squid_curtime - 86400) {
> > > ++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot
> > > store "
> > > << cap << " entries. Limiting to " << absolute_max);
> > > ++ last_loud = squid_curtime;
> > > ++ } else {
> > > ++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap
> > > << "
> > > entries. Limiting to " << absolute_max);
> > > ++ }
> > > ++ cap = absolute_max;
> > > ++ }
> > > ++
> > > ++ return cap;
> > > ++}
> > > + #endif /* USE_CACHE_DIGESTS */
> > > +
> > > +-static void
> > > +-storeDigestRegisterWithCacheManager(void)
> > > ++void
> > > ++storeDigestInit(void)
> > > + {
> > > + Mgr::RegisterAction("store_digest", "Store Digest",
> > > storeDigestReport,
> > > 0, 1);
> > > +-}
> > > +-
> > > +-/*
> > > +- * PUBLIC FUNCTIONS
> > > +- */
> > > +-
> > > +-void
> > > +-storeDigestInit(void)
> > > +-{
> > > +- storeDigestRegisterWithCacheManager();
> > > +
> > > + #if USE_CACHE_DIGESTS
> > > +- const int cap = storeDigestCalcCap();
> > > +-
> > > + if (!Config.onoff.digest_generation) {
> > > + store_digest = NULL;
> > > + debugs(71, 3, "Local cache digest generation disabled");
> > > + return;
> > > + }
> > > +
> > > ++ const uint64_t cap = storeDigestCalcCap();
> > > + store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
> > > + debugs(71, DBG_IMPORTANT, "Local cache digest enabled;
> > > rebuild/rewrite
> > > every " <<
> > > + (int) Config.digest.rebuild_period << "/" <<
> > > +@@ -290,6 +317,31 @@
> > > + storeDigestRebuildResume();
> > > + }
> > > +
> > > ++/// \returns true if we actually resized the digest
> > > ++static bool
> > > ++storeDigestResize()
> > > ++{
> > > ++ const uint64_t cap = storeDigestCalcCap();
> > > ++ assert(store_digest);
> > > ++ uint64_t diff;
> > > ++ if (cap > store_digest->capacity)
> > > ++ diff = cap - store_digest->capacity;
> > > ++ else
> > > ++ diff = store_digest->capacity - cap;
> > > ++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change:
> > > " <<
> > > ++ diff << " (" << xpercentInt(diff, store_digest->capacity) <<
> > > "%)"
> > > );
> > > ++ /* avoid minor adjustments */
> > > ++
> > > ++ if (diff <= store_digest->capacity / 10) {
> > > ++ debugs(71, 2, "small change, will not resize.");
> > > ++ return false;
> > > ++ } else {
> > > ++ debugs(71, 2, "big change, resizing.");
> > > ++ cacheDigestChangeCap(store_digest, cap);
> > > ++ }
> > > ++ return true;
> > > ++}
> > > ++
> > > + /* called be Rewrite to push Rebuild forward */
> > > + static void
> > > + storeDigestRebuildResume(void)
> > > +@@ -439,7 +491,7 @@
> > > + assert(e);
> > > + /* _add_ check that nothing bad happened while we were waiting @?@
> > > @?@
> > > */
> > > +
> > > +- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
> > > ++ if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) >
> > > store_digest->mask_size)
> > > + chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
> > > +
> > > + e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
> > > +@@ -451,7 +503,7 @@
> > > + sd_state.rewrite_offset += chunk_size;
> > > +
> > > + /* are we done ? */
> > > +- if (sd_state.rewrite_offset >= store_digest->mask_size)
> > > ++ if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest-
> > > > mask_size)
> > > + storeDigestRewriteFinish(e);
> > > + else
> > > + eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data,
> > > 0.0, 1, false);
> > > +@@ -467,60 +519,10 @@
> > > + sd_state.cblock.count = htonl(store_digest->count);
> > > + sd_state.cblock.del_count = htonl(store_digest->del_count);
> > > + sd_state.cblock.mask_size = htonl(store_digest->mask_size);
> > > +- sd_state.cblock.bits_per_entry = (unsigned char)
> > > +- Config.digest.bits_per_entry;
> > > ++ sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
> > > + sd_state.cblock.hash_func_count = (unsigned char)
> > > CacheDigestHashFuncCount;
> > > + e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
> > > + }
> > > +
> > > +-/* calculates digest capacity */
> > > +-static int
> > > +-storeDigestCalcCap(void)
> > > +-{
> > > +- /*
> > > +- * To-Do: Bloom proved that the optimal filter utilization is 50%
> > > (half
> > > of
> > > +- * the bits are off). However, we do not have a formula to calculate
> > > the
> > > +- * number of _entries_ we want to pre-allocate for.
> > > +- */
> > > +- const int hi_cap = Store::Root().maxSize() /
> > > Config.Store.avgObjectSize;
> > > +- const int lo_cap = 1 + Store::Root().currentSize() /
> > > Config.Store.avgObjectSize;
> > > +- const int e_count = StoreEntry::inUseCount();
> > > +- int cap = e_count ? e_count :hi_cap;
> > > +- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want "
> > > <<
> > > cap <<
> > > +- " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
> > > +-
> > > +- if (cap < lo_cap)
> > > +- cap = lo_cap;
> > > +-
> > > +- /* do not enforce hi_cap limit, average-based estimation may be
> > > wrong
> > > +- *if (cap > hi_cap)
> > > +- * cap = hi_cap;
> > > +- */
> > > +- return cap;
> > > +-}
> > > +-
> > > +-/* returns true if we actually resized the digest */
> > > +-static int
> > > +-storeDigestResize(void)
> > > +-{
> > > +- const int cap = storeDigestCalcCap();
> > > +- int diff;
> > > +- assert(store_digest);
> > > +- diff = abs(cap - store_digest->capacity);
> > > +- debugs(71, 2, "storeDigestResize: " <<
> > > +- store_digest->capacity << " -> " << cap << "; change: " <<
> > > +- diff << " (" << xpercentInt(diff, store_digest->capacity) <<
> > > "%)"
> > > );
> > > +- /* avoid minor adjustments */
> > > +-
> > > +- if (diff <= store_digest->capacity / 10) {
> > > +- debugs(71, 2, "storeDigestResize: small change, will not
> > > resize.");
> > > +- return 0;
> > > +- } else {
> > > +- debugs(71, 2, "storeDigestResize: big change, resizing.");
> > > +- cacheDigestChangeCap(store_digest, cap);
> > > +- return 1;
> > > +- }
> > > +-}
> > > +-
> > > + #endif /* USE_CACHE_DIGESTS */
> > > +
> > > +
> > > +=== modified file 'src/tests/stub_CacheDigest.cc'
> > > +--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
> > > ++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
> > > +@@ -16,11 +16,11 @@
> > > + class CacheDigestGuessStats;
> > > + class StoreEntry;
> > > +
> > > +-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
> > > ++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
> > > + void cacheDigestDestroy(CacheDigest *) STUB
> > > + CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
> > > + void cacheDigestClear(CacheDigest * ) STUB
> > > +-void cacheDigestChangeCap(CacheDigest *,int) STUB
> > > ++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
> > > + int cacheDigestTest(const CacheDigest *, const cache_key *)
> > > STUB_RETVAL(1)
> > > + void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
> > > + void cacheDigestDel(CacheDigest *, const cache_key *) STUB
> > > +@@ -28,5 +28,4 @@
> > > + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *,
> > > StoreEntry
> > > *, const char *) STUB
> > > + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
> > > +-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
> > > +-
> > > ++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
> > > +
> > > diff --git a/src/patches/squid/squid-3.5-14068.patch
> > > b/src/patches/squid/squid-3.5-14068.patch
> > > new file mode 100644
> > > index 0000000..4766e00
> > > --- /dev/null
> > > +++ b/src/patches/squid/squid-3.5-14068.patch
> > > @@ -0,0 +1,35 @@
> > > +------------------------------------------------------------
> > > +revno: 14068
> > > +revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> > > +parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
> > > +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
> > > +author: Anonymous <bigparrot(a)pirateperfection.com>
> > > +committer: Amos Jeffries <squid3(a)treenet.co.nz>
> > > +branch nick: 3.5
> > > +timestamp: Sat 2016-07-23 19:19:30 +1200
> > > +message:
> > > + Bug #4542: authentication credentials IP TTL updated incorrectly
> > > +------------------------------------------------------------
> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
> > > +# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
> > > +# timestamp: 2016-07-23 07:24:05 +0000
> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
> > > +# 1wzqpbyi1rk5w6vg
> > > +#
> > > +# Begin patch
> > > +=== modified file 'src/auth/User.cc'
> > > +--- src/auth/User.cc 2016-01-01 00:14:27 +0000
> > > ++++ src/auth/User.cc 2016-07-23 07:19:30 +0000
> > > +@@ -284,7 +284,7 @@
> > > + /* This ip has already been seen. */
> > > + found = 1;
> > > + /* update IP ttl */
> > > +- ipdata->ip_expiretime = squid_curtime;
> > > ++ ipdata->ip_expiretime = squid_curtime +
> > > ::Config.authenticateIpTTL;
> > > + } else if (ipdata->ip_expiretime <= squid_curtime) {
> > > + /* This IP has expired - remove from the seen list */
> > > + dlinkDelete(&ipdata->node, &ip_list);
> > > +
> > > diff --git a/src/patches/squid/squid-3.5-14069.patch
> > > b/src/patches/squid/squid-3.5-14069.patch
> > > new file mode 100644
> > > index 0000000..15ca37a
> > > --- /dev/null
> > > +++ b/src/patches/squid/squid-3.5-14069.patch
> > > @@ -0,0 +1,30 @@
> > > +------------------------------------------------------------
> > > +revno: 14069
> > > +revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
> > > +parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
> > > +committer: Source Maintenance <squidadm(a)squid-cache.org>
> > > +branch nick: 3.5
> > > +timestamp: Sat 2016-07-23 12:13:51 +0000
> > > +message:
> > > + SourceFormat Enforcement
> > > +------------------------------------------------------------
> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
> > > +# revision_id: squidadm(a)squid-cache.org-20160723121351-\
> > > +# iuc8hwstrqd0l1dv
> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
> > > +# timestamp: 2016-07-23 12:50:56 +0000
> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
> > > +# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
> > > +# cemledcltg8pkc28
> > > +#
> > > +# Begin patch
> > > +=== modified file 'src/tests/stub_CacheDigest.cc'
> > > +--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
> > > ++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000
> > > +@@ -29,3 +29,4 @@
> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *,
> > > StoreEntry
> > > *, const char *) STUB
> > > + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
> > > + uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
> > > ++
> > > +
> >
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-08-03 10:47 ` Michael Tremer
@ 2016-08-03 18:49 ` Matthias Fischer
2016-08-18 16:07 ` Matthias Fischer
1 sibling, 0 replies; 7+ messages in thread
From: Matthias Fischer @ 2016-08-03 18:49 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 26141 bytes --]
Hi,
On 03.08.2016 12:47, Michael Tremer wrote:
> Hi,
>
> I didn't check the source, yet because it fails to apply before that:
This is the first time I see this kind of error...
> [root(a)rice-oxley ipfire-2.x]# pwclient git-am -s 692
> Applying patch #692 using 'git am -s'
> Description: squid 3.5.20: latest patches from upstream
> Applying: squid 3.5.20: latest patches from upstream
> .git/rebase-apply/patch:50: trailing whitespace.
> #
> .git/rebase-apply/patch:57: trailing whitespace.
>
> .git/rebase-apply/patch:73: trailing whitespace.
>
> .git/rebase-apply/patch:81: trailing whitespace.
>
> .git/rebase-apply/patch:107: trailing whitespace.
>
> error: patch failed: lfs/squid:70
> error: lfs/squid: patch does not apply
> Patch failed at 0001 squid 3.5.20: latest patches from upstream
> The copy of the patch that failed is found in: .git/rebase-apply/patch
> When you have resolved this problem, run "git am --continue".
> If you prefer to skip this patch, run "git am --skip" instead.
> To restore the original branch and stop patching, run "git am --abort".
> 'git am' failed with exit status 128
Sorry: I tried, but didn't find the error. And 'git diff --check' didn't
throw any warnings or errors.
The patch files are exactly as I pulled them from
http://www.squid-cache.org/Versions/v3/3.5/changesets/
and I have no further idea where to look at, because on my 'Devel' these
three patches apply without any errors.
Any clue where to look at?
Best,
Matthias
> On Tue, 2016-08-02 at 20:49 +0200, Matthias Fischer wrote:
>> Hi,
>>
>> On 02.08.2016 17:14, Michael Tremer wrote:
>> > This one won't apply onto next... :(
>> >
>> > Could you please check why?
>>
>> Its always the same here (*sigh*): sources seemed to be missing.
>>
>> Please check again. Its running here.
>>
>> Best,
>> Matthias
>>
>> > On Sun, 2016-07-24 at 13:44 +0200, Matthias Fischer wrote:
>> > > Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
>> > > ---
>> > > lfs/squid | 3 +
>> > > src/patches/squid/squid-3.5-14067.patch | 381
>> > > ++++++++++++++++++++++++++++++++
>> > > src/patches/squid/squid-3.5-14068.patch | 35 +++
>> > > src/patches/squid/squid-3.5-14069.patch | 30 +++
>> > > 4 files changed, 449 insertions(+)
>> > > create mode 100644 src/patches/squid/squid-3.5-14067.patch
>> > > create mode 100644 src/patches/squid/squid-3.5-14068.patch
>> > > create mode 100644 src/patches/squid/squid-3.5-14069.patch
>> > >
>> > > diff --git a/lfs/squid b/lfs/squid
>> > > index 5bc976e..a88f0c5 100644
>> > > --- a/lfs/squid
>> > > +++ b/lfs/squid
>> > > @@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
>> > > $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
>> > > @$(PREBUILD)
>> > > @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf
>> > > $(DIR_DL)/$(DL_FILE)
>> > > + cd $(DIR_APP) && patch -Np0 -i
>> > > $(DIR_SRC)/src/patches/squid/squid-
>> > > 3.5-14067.patch
>> > > + cd $(DIR_APP) && patch -Np0 -i
>> > > $(DIR_SRC)/src/patches/squid/squid-
>> > > 3.5-14068.patch
>> > > + cd $(DIR_APP) && patch -Np0 -i
>> > > $(DIR_SRC)/src/patches/squid/squid-
>> > > 3.5-14069.patch
>> > > cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-
>> > > 3.5.17-
>> > > fix-max-file-descriptors.patch
>> > >
>> > > cd $(DIR_APP) && autoreconf -vfi
>> > > diff --git a/src/patches/squid/squid-3.5-14067.patch
>> > > b/src/patches/squid/squid-3.5-14067.patch
>> > > new file mode 100644
>> > > index 0000000..8d9cb21
>> > > --- /dev/null
>> > > +++ b/src/patches/squid/squid-3.5-14067.patch
>> > > @@ -0,0 +1,381 @@
>> > > +------------------------------------------------------------
>> > > +revno: 14067
>> > > +revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> > > +parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
>> > > +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
>> > > +committer: Amos Jeffries <squid3(a)treenet.co.nz>
>> > > +branch nick: 3.5
>> > > +timestamp: Sat 2016-07-23 19:16:20 +1200
>> > > +message:
>> > > + Bug 4534: assertion failure in xcalloc when using many cache_dir
>> > > +------------------------------------------------------------
>> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
>> > > +# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
>> > > +# timestamp: 2016-07-23 07:24:01 +0000
>> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
>> > > +# vpjak1pq4uecadd2
>> > > +#
>> > > +# Begin patch
>> > > +=== modified file 'src/CacheDigest.cc'
>> > > +--- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
>> > > ++++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
>> > > +@@ -35,12 +35,12 @@
>> > > + static uint32_t hashed_keys[4];
>> > > +
>> > > + static void
>> > > +-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
>> > > ++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
>> > > + {
>> > > +- const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
>> > > ++ const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
>> > > + assert(cd);
>> > > + assert(capacity > 0 && bpe > 0);
>> > > +- assert(mask_size > 0);
>> > > ++ assert(mask_size != 0);
>> > > + cd->capacity = capacity;
>> > > + cd->bits_per_entry = bpe;
>> > > + cd->mask_size = mask_size;
>> > > +@@ -50,7 +50,7 @@
>> > > + }
>> > > +
>> > > + CacheDigest *
>> > > +-cacheDigestCreate(int capacity, int bpe)
>> > > ++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
>> > > + {
>> > > + CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
>> > > + assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely
>> > > on 16
>> > > byte keys */
>> > > +@@ -97,7 +97,7 @@
>> > > +
>> > > + /* changes mask size, resets bits to 0, preserves "cd" pointer */
>> > > + void
>> > > +-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
>> > > ++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
>> > > + {
>> > > + assert(cd);
>> > > + cacheDigestClean(cd);
>> > > +@@ -278,12 +278,12 @@
>> > > + storeAppendPrintf(e, "%s digest: size: %d bytes\n",
>> > > + label ? label : "", stats.bit_count / 8
>> > > + );
>> > > +- storeAppendPrintf(e, "\t entries: count: %d capacity: %d util:
>> > > %d%%\n",
>> > > ++ storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %"
>> > > PRIu64
>> > > " util: %d%%\n",
>> > > + cd->count,
>> > > + cd->capacity,
>> > > + xpercentInt(cd->count, cd->capacity)
>> > > + );
>> > > +- storeAppendPrintf(e, "\t deletion attempts: %d\n",
>> > > ++ storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
>> > > + cd->del_count
>> > > + );
>> > > + storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d
>> > > util:
>> > > %d%%\n",
>> > > +@@ -297,16 +297,18 @@
>> > > + );
>> > > + }
>> > > +
>> > > +-size_t
>> > > +-cacheDigestCalcMaskSize(int cap, int bpe)
>> > > ++uint32_t
>> > > ++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
>> > > + {
>> > > +- return (size_t) (cap * bpe + 7) / 8;
>> > > ++ uint64_t bitCount = (cap * bpe) + 7;
>> > > ++ assert(bitCount < INT_MAX); // dont 31-bit overflow later
>> > > ++ return static_cast<uint32_t>(bitCount / 8);
>> > > + }
>> > > +
>> > > + static void
>> > > + cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
>> > > + {
>> > > +- const unsigned int bit_count = cd->mask_size * 8;
>> > > ++ const uint32_t bit_count = cd->mask_size * 8;
>> > > + unsigned int tmp_keys[4];
>> > > + /* we must memcpy to ensure alignment */
>> > > + memcpy(tmp_keys, key, sizeof(tmp_keys));
>> > > +
>> > > +=== modified file 'src/CacheDigest.h'
>> > > +--- src/CacheDigest.h 2016-01-01 00:14:27 +0000
>> > > ++++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
>> > > +@@ -22,23 +22,23 @@
>> > > + {
>> > > + public:
>> > > + /* public, read-only */
>> > > +- char *mask; /* bit mask */
>> > > +- int mask_size; /* mask size in bytes */
>> > > +- int capacity; /* expected maximum for .count, not a hard limit
>> > > */
>> > > +- int bits_per_entry; /* number of bits allocated for each entry
>> > > from
>> > > capacity */
>> > > +- int count; /* number of digested entries */
>> > > +- int del_count; /* number of deletions performed so far */
>> > > ++ uint64_t count; /* number of digested entries */
>> > > ++ uint64_t del_count; /* number of deletions performed so far */
>> > > ++ uint64_t capacity; /* expected maximum for .count, not a hard
>> > > limit */
>> > > ++ char *mask; /* bit mask */
>> > > ++ uint32_t mask_size; /* mask size in bytes */
>> > > ++ int8_t bits_per_entry; /* number of bits allocated for each entry
>> > > from
>> > > capacity */
>> > > + };
>> > > +
>> > > +-CacheDigest *cacheDigestCreate(int capacity, int bpe);
>> > > ++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
>> > > + void cacheDigestDestroy(CacheDigest * cd);
>> > > + CacheDigest *cacheDigestClone(const CacheDigest * cd);
>> > > + void cacheDigestClear(CacheDigest * cd);
>> > > +-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
>> > > ++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
>> > > + int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
>> > > + void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
>> > > + void cacheDigestDel(CacheDigest * cd, const cache_key * key);
>> > > +-size_t cacheDigestCalcMaskSize(int cap, int bpe);
>> > > ++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
>> > > + int cacheDigestBitUtil(const CacheDigest * cd);
>> > > + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int
>> > > real_hit, int guess_hit);
>> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats,
>> > > StoreEntry * sentry, const char *label);
>> > > +
>> > > +=== modified file 'src/PeerDigest.h'
>> > > +--- src/PeerDigest.h 2016-01-01 00:14:27 +0000
>> > > ++++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
>> > > +@@ -52,7 +52,7 @@
>> > > + store_client *old_sc;
>> > > + HttpRequest *request;
>> > > + int offset;
>> > > +- int mask_offset;
>> > > ++ uint32_t mask_offset;
>> > > + time_t start_time;
>> > > + time_t resp_time;
>> > > + time_t expires;
>> > > +
>> > > +=== modified file 'src/peer_digest.cc'
>> > > +--- src/peer_digest.cc 2016-01-01 00:14:27 +0000
>> > > ++++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
>> > > +@@ -754,7 +754,7 @@
>> > > + if (!reason && !size) {
>> > > + if (!pd->cd)
>> > > + reason = "null digest?!";
>> > > +- else if (fetch->mask_offset != (int)pd->cd->mask_size)
>> > > ++ else if (fetch->mask_offset != pd->cd->mask_size)
>> > > + reason = "premature end of digest?!";
>> > > + else if (!peerDigestUseful(pd))
>> > > + reason = "useless digest";
>> > > +
>> > > +=== modified file 'src/store_digest.cc'
>> > > +--- src/store_digest.cc 2016-01-01 00:14:27 +0000
>> > > ++++ src/store_digest.cc 2016-07-23 07:16:20 +0000
>> > > +@@ -76,36 +76,63 @@
>> > > + static void storeDigestRewriteFinish(StoreEntry * e);
>> > > + static EVH storeDigestSwapOutStep;
>> > > + static void storeDigestCBlockSwapOut(StoreEntry * e);
>> > > +-static int storeDigestCalcCap(void);
>> > > +-static int storeDigestResize(void);
>> > > + static void storeDigestAdd(const StoreEntry *);
>> > > +
>> > > ++/// calculates digest capacity
>> > > ++static uint64_t
>> > > ++storeDigestCalcCap()
>> > > ++{
>> > > ++ /*
>> > > ++ * To-Do: Bloom proved that the optimal filter utilization is 50%
>> > > (half
>> > > of
>> > > ++ * the bits are off). However, we do not have a formula to calculate
>> > > the
>> > > ++ * number of _entries_ we want to pre-allocate for.
>> > > ++ */
>> > > ++ const uint64_t hi_cap = Store::Root().maxSize() /
>> > > Config.Store.avgObjectSize;
>> > > ++ const uint64_t lo_cap = 1 + Store::Root().currentSize() /
>> > > Config.Store.avgObjectSize;
>> > > ++ const uint64_t e_count = StoreEntry::inUseCount();
>> > > ++ uint64_t cap = e_count ? e_count : hi_cap;
>> > > ++ debugs(71, 2, "have: " << e_count << ", want " << cap <<
>> > > ++ " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
>> > > ++
>> > > ++ if (cap < lo_cap)
>> > > ++ cap = lo_cap;
>> > > ++
>> > > ++ /* do not enforce hi_cap limit, average-based estimation may be
>> > > wrong
>> > > ++ *if (cap > hi_cap)
>> > > ++ * cap = hi_cap;
>> > > ++ */
>> > > ++
>> > > ++ // Bug 4534: we still have to set an upper-limit at some reasonable
>> > > value though.
>> > > ++ // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
>> > > ++ const uint64_t absolute_max = (INT_MAX -8) /
>> > > Config.digest.bits_per_entry;
>> > > ++ if (cap > absolute_max) {
>> > > ++ static time_t last_loud = 0;
>> > > ++ if (last_loud < squid_curtime - 86400) {
>> > > ++ debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot
>> > > store "
>> > > << cap << " entries. Limiting to " << absolute_max);
>> > > ++ last_loud = squid_curtime;
>> > > ++ } else {
>> > > ++ debugs(71, 3, "WARNING: Cache Digest cannot store " << cap
>> > > << "
>> > > entries. Limiting to " << absolute_max);
>> > > ++ }
>> > > ++ cap = absolute_max;
>> > > ++ }
>> > > ++
>> > > ++ return cap;
>> > > ++}
>> > > + #endif /* USE_CACHE_DIGESTS */
>> > > +
>> > > +-static void
>> > > +-storeDigestRegisterWithCacheManager(void)
>> > > ++void
>> > > ++storeDigestInit(void)
>> > > + {
>> > > + Mgr::RegisterAction("store_digest", "Store Digest",
>> > > storeDigestReport,
>> > > 0, 1);
>> > > +-}
>> > > +-
>> > > +-/*
>> > > +- * PUBLIC FUNCTIONS
>> > > +- */
>> > > +-
>> > > +-void
>> > > +-storeDigestInit(void)
>> > > +-{
>> > > +- storeDigestRegisterWithCacheManager();
>> > > +
>> > > + #if USE_CACHE_DIGESTS
>> > > +- const int cap = storeDigestCalcCap();
>> > > +-
>> > > + if (!Config.onoff.digest_generation) {
>> > > + store_digest = NULL;
>> > > + debugs(71, 3, "Local cache digest generation disabled");
>> > > + return;
>> > > + }
>> > > +
>> > > ++ const uint64_t cap = storeDigestCalcCap();
>> > > + store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
>> > > + debugs(71, DBG_IMPORTANT, "Local cache digest enabled;
>> > > rebuild/rewrite
>> > > every " <<
>> > > + (int) Config.digest.rebuild_period << "/" <<
>> > > +@@ -290,6 +317,31 @@
>> > > + storeDigestRebuildResume();
>> > > + }
>> > > +
>> > > ++/// \returns true if we actually resized the digest
>> > > ++static bool
>> > > ++storeDigestResize()
>> > > ++{
>> > > ++ const uint64_t cap = storeDigestCalcCap();
>> > > ++ assert(store_digest);
>> > > ++ uint64_t diff;
>> > > ++ if (cap > store_digest->capacity)
>> > > ++ diff = cap - store_digest->capacity;
>> > > ++ else
>> > > ++ diff = store_digest->capacity - cap;
>> > > ++ debugs(71, 2, store_digest->capacity << " -> " << cap << "; change:
>> > > " <<
>> > > ++ diff << " (" << xpercentInt(diff, store_digest->capacity) <<
>> > > "%)"
>> > > );
>> > > ++ /* avoid minor adjustments */
>> > > ++
>> > > ++ if (diff <= store_digest->capacity / 10) {
>> > > ++ debugs(71, 2, "small change, will not resize.");
>> > > ++ return false;
>> > > ++ } else {
>> > > ++ debugs(71, 2, "big change, resizing.");
>> > > ++ cacheDigestChangeCap(store_digest, cap);
>> > > ++ }
>> > > ++ return true;
>> > > ++}
>> > > ++
>> > > + /* called be Rewrite to push Rebuild forward */
>> > > + static void
>> > > + storeDigestRebuildResume(void)
>> > > +@@ -439,7 +491,7 @@
>> > > + assert(e);
>> > > + /* _add_ check that nothing bad happened while we were waiting @?@
>> > > @?@
>> > > */
>> > > +
>> > > +- if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
>> > > ++ if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) >
>> > > store_digest->mask_size)
>> > > + chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
>> > > +
>> > > + e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
>> > > +@@ -451,7 +503,7 @@
>> > > + sd_state.rewrite_offset += chunk_size;
>> > > +
>> > > + /* are we done ? */
>> > > +- if (sd_state.rewrite_offset >= store_digest->mask_size)
>> > > ++ if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest-
>> > > > mask_size)
>> > > + storeDigestRewriteFinish(e);
>> > > + else
>> > > + eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data,
>> > > 0.0, 1, false);
>> > > +@@ -467,60 +519,10 @@
>> > > + sd_state.cblock.count = htonl(store_digest->count);
>> > > + sd_state.cblock.del_count = htonl(store_digest->del_count);
>> > > + sd_state.cblock.mask_size = htonl(store_digest->mask_size);
>> > > +- sd_state.cblock.bits_per_entry = (unsigned char)
>> > > +- Config.digest.bits_per_entry;
>> > > ++ sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
>> > > + sd_state.cblock.hash_func_count = (unsigned char)
>> > > CacheDigestHashFuncCount;
>> > > + e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
>> > > + }
>> > > +
>> > > +-/* calculates digest capacity */
>> > > +-static int
>> > > +-storeDigestCalcCap(void)
>> > > +-{
>> > > +- /*
>> > > +- * To-Do: Bloom proved that the optimal filter utilization is 50%
>> > > (half
>> > > of
>> > > +- * the bits are off). However, we do not have a formula to calculate
>> > > the
>> > > +- * number of _entries_ we want to pre-allocate for.
>> > > +- */
>> > > +- const int hi_cap = Store::Root().maxSize() /
>> > > Config.Store.avgObjectSize;
>> > > +- const int lo_cap = 1 + Store::Root().currentSize() /
>> > > Config.Store.avgObjectSize;
>> > > +- const int e_count = StoreEntry::inUseCount();
>> > > +- int cap = e_count ? e_count :hi_cap;
>> > > +- debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want "
>> > > <<
>> > > cap <<
>> > > +- " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
>> > > +-
>> > > +- if (cap < lo_cap)
>> > > +- cap = lo_cap;
>> > > +-
>> > > +- /* do not enforce hi_cap limit, average-based estimation may be
>> > > wrong
>> > > +- *if (cap > hi_cap)
>> > > +- * cap = hi_cap;
>> > > +- */
>> > > +- return cap;
>> > > +-}
>> > > +-
>> > > +-/* returns true if we actually resized the digest */
>> > > +-static int
>> > > +-storeDigestResize(void)
>> > > +-{
>> > > +- const int cap = storeDigestCalcCap();
>> > > +- int diff;
>> > > +- assert(store_digest);
>> > > +- diff = abs(cap - store_digest->capacity);
>> > > +- debugs(71, 2, "storeDigestResize: " <<
>> > > +- store_digest->capacity << " -> " << cap << "; change: " <<
>> > > +- diff << " (" << xpercentInt(diff, store_digest->capacity) <<
>> > > "%)"
>> > > );
>> > > +- /* avoid minor adjustments */
>> > > +-
>> > > +- if (diff <= store_digest->capacity / 10) {
>> > > +- debugs(71, 2, "storeDigestResize: small change, will not
>> > > resize.");
>> > > +- return 0;
>> > > +- } else {
>> > > +- debugs(71, 2, "storeDigestResize: big change, resizing.");
>> > > +- cacheDigestChangeCap(store_digest, cap);
>> > > +- return 1;
>> > > +- }
>> > > +-}
>> > > +-
>> > > + #endif /* USE_CACHE_DIGESTS */
>> > > +
>> > > +
>> > > +=== modified file 'src/tests/stub_CacheDigest.cc'
>> > > +--- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
>> > > ++++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
>> > > +@@ -16,11 +16,11 @@
>> > > + class CacheDigestGuessStats;
>> > > + class StoreEntry;
>> > > +
>> > > +-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
>> > > ++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
>> > > + void cacheDigestDestroy(CacheDigest *) STUB
>> > > + CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
>> > > + void cacheDigestClear(CacheDigest * ) STUB
>> > > +-void cacheDigestChangeCap(CacheDigest *,int) STUB
>> > > ++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
>> > > + int cacheDigestTest(const CacheDigest *, const cache_key *)
>> > > STUB_RETVAL(1)
>> > > + void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
>> > > + void cacheDigestDel(CacheDigest *, const cache_key *) STUB
>> > > +@@ -28,5 +28,4 @@
>> > > + void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
>> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *,
>> > > StoreEntry
>> > > *, const char *) STUB
>> > > + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
>> > > +-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
>> > > +-
>> > > ++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
>> > > +
>> > > diff --git a/src/patches/squid/squid-3.5-14068.patch
>> > > b/src/patches/squid/squid-3.5-14068.patch
>> > > new file mode 100644
>> > > index 0000000..4766e00
>> > > --- /dev/null
>> > > +++ b/src/patches/squid/squid-3.5-14068.patch
>> > > @@ -0,0 +1,35 @@
>> > > +------------------------------------------------------------
>> > > +revno: 14068
>> > > +revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> > > +parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
>> > > +fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
>> > > +author: Anonymous <bigparrot(a)pirateperfection.com>
>> > > +committer: Amos Jeffries <squid3(a)treenet.co.nz>
>> > > +branch nick: 3.5
>> > > +timestamp: Sat 2016-07-23 19:19:30 +1200
>> > > +message:
>> > > + Bug #4542: authentication credentials IP TTL updated incorrectly
>> > > +------------------------------------------------------------
>> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
>> > > +# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
>> > > +# timestamp: 2016-07-23 07:24:05 +0000
>> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
>> > > +# 1wzqpbyi1rk5w6vg
>> > > +#
>> > > +# Begin patch
>> > > +=== modified file 'src/auth/User.cc'
>> > > +--- src/auth/User.cc 2016-01-01 00:14:27 +0000
>> > > ++++ src/auth/User.cc 2016-07-23 07:19:30 +0000
>> > > +@@ -284,7 +284,7 @@
>> > > + /* This ip has already been seen. */
>> > > + found = 1;
>> > > + /* update IP ttl */
>> > > +- ipdata->ip_expiretime = squid_curtime;
>> > > ++ ipdata->ip_expiretime = squid_curtime +
>> > > ::Config.authenticateIpTTL;
>> > > + } else if (ipdata->ip_expiretime <= squid_curtime) {
>> > > + /* This IP has expired - remove from the seen list */
>> > > + dlinkDelete(&ipdata->node, &ip_list);
>> > > +
>> > > diff --git a/src/patches/squid/squid-3.5-14069.patch
>> > > b/src/patches/squid/squid-3.5-14069.patch
>> > > new file mode 100644
>> > > index 0000000..15ca37a
>> > > --- /dev/null
>> > > +++ b/src/patches/squid/squid-3.5-14069.patch
>> > > @@ -0,0 +1,30 @@
>> > > +------------------------------------------------------------
>> > > +revno: 14069
>> > > +revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
>> > > +parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
>> > > +committer: Source Maintenance <squidadm(a)squid-cache.org>
>> > > +branch nick: 3.5
>> > > +timestamp: Sat 2016-07-23 12:13:51 +0000
>> > > +message:
>> > > + SourceFormat Enforcement
>> > > +------------------------------------------------------------
>> > > +# Bazaar merge directive format 2 (Bazaar 0.90)
>> > > +# revision_id: squidadm(a)squid-cache.org-20160723121351-\
>> > > +# iuc8hwstrqd0l1dv
>> > > +# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
>> > > +# timestamp: 2016-07-23 12:50:56 +0000
>> > > +# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
>> > > +# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
>> > > +# cemledcltg8pkc28
>> > > +#
>> > > +# Begin patch
>> > > +=== modified file 'src/tests/stub_CacheDigest.cc'
>> > > +--- src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
>> > > ++++ src/tests/stub_CacheDigest.cc 2016-07-23 12:13:51 +0000
>> > > +@@ -29,3 +29,4 @@
>> > > + void cacheDigestGuessStatsReport(const CacheDigestGuessStats *,
>> > > StoreEntry
>> > > *, const char *) STUB
>> > > + void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
>> > > + uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
>> > > ++
>> > > +
>> >
>>
>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-08-03 10:47 ` Michael Tremer
2016-08-03 18:49 ` Matthias Fischer
@ 2016-08-18 16:07 ` Matthias Fischer
2016-08-18 17:26 ` Michael Tremer
1 sibling, 1 reply; 7+ messages in thread
From: Matthias Fischer @ 2016-08-18 16:07 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 1219 bytes --]
Hi,
On 03.08.2016 12:47, Michael Tremer wrote:
> Hi,
>
> I didn't check the source, yet because it fails to apply before that:
>
> [root(a)rice-oxley ipfire-2.x]# pwclient git-am -s 692
> Applying patch #692 using 'git am -s'
> Description: squid 3.5.20: latest patches from upstream
> Applying: squid 3.5.20: latest patches from upstream
> .git/rebase-apply/patch:50: trailing whitespace.
> #
> .git/rebase-apply/patch:57: trailing whitespace.
>
> .git/rebase-apply/patch:73: trailing whitespace.
>
> .git/rebase-apply/patch:81: trailing whitespace.
>
> .git/rebase-apply/patch:107: trailing whitespace.
>
> error: patch failed: lfs/squid:70
> error: lfs/squid: patch does not apply
> Patch failed at 0001 squid 3.5.20: latest patches from upstream
> The copy of the patch that failed is found in: .git/rebase-apply/patch
> When you have resolved this problem, run "git am --continue".
> If you prefer to skip this patch, run "git am --skip" instead.
> To restore the original branch and stop patching, run "git am --abort".
> 'git am' failed with exit status 128
> ...
Being curious:
Anything new about this problem?
I doublechecked my sources - here its building without any problems.
Best,
Matthias
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] squid 3.5.20: latest patches from upstream
2016-08-18 16:07 ` Matthias Fischer
@ 2016-08-18 17:26 ` Michael Tremer
0 siblings, 0 replies; 7+ messages in thread
From: Michael Tremer @ 2016-08-18 17:26 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 1506 bytes --]
Hi,
actually not.
Could you just upload the entire branch with those changes and I will
have a look at it?
Best,
-Michael
On Thu, 2016-08-18 at 18:07 +0200, Matthias Fischer wrote:
> Hi,
>
> On 03.08.2016 12:47, Michael Tremer wrote:
> >
> > Hi,
> >
> > I didn't check the source, yet because it fails to apply before
> > that:
> >
> > [root(a)rice-oxley ipfire-2.x]# pwclient git-am -s 692
> > Applying patch #692 using 'git am -s'
> > Description: squid 3.5.20: latest patches from upstream
> > Applying: squid 3.5.20: latest patches from upstream
> > .git/rebase-apply/patch:50: trailing whitespace.
> > #
> > .git/rebase-apply/patch:57: trailing whitespace.
> >
> > .git/rebase-apply/patch:73: trailing whitespace.
> >
> > .git/rebase-apply/patch:81: trailing whitespace.
> >
> > .git/rebase-apply/patch:107: trailing whitespace.
> >
> > error: patch failed: lfs/squid:70
> > error: lfs/squid: patch does not apply
> > Patch failed at 0001 squid 3.5.20: latest patches from upstream
> > The copy of the patch that failed is found in: .git/rebase-
> > apply/patch
> > When you have resolved this problem, run "git am --continue".
> > If you prefer to skip this patch, run "git am --skip" instead.
> > To restore the original branch and stop patching, run "git am --
> > abort".
> > 'git am' failed with exit status 128
> > ...
>
> Being curious:
> Anything new about this problem?
>
> I doublechecked my sources - here its building without any problems.
>
> Best,
> Matthias
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2016-08-18 17:26 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-07-24 11:44 [PATCH] squid 3.5.20: latest patches from upstream Matthias Fischer
2016-08-02 15:14 ` Michael Tremer
2016-08-02 18:49 ` Matthias Fischer
2016-08-03 10:47 ` Michael Tremer
2016-08-03 18:49 ` Matthias Fischer
2016-08-18 16:07 ` Matthias Fischer
2016-08-18 17:26 ` Michael Tremer
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox