[PATCH] squid 3.5.20: latest patches from upstream

[Matthias Fischer <matthias.fischer@ipfire.org>]

[-- Attachment #1: Type: text/plain, Size: 20182 bytes --]

Signed-off-by: Matthias Fischer <matthias.fischer(a)ipfire.org>
---
 lfs/squid                               |   3 +
 src/patches/squid/squid-3.5-14067.patch | 381 ++++++++++++++++++++++++++++++++
 src/patches/squid/squid-3.5-14068.patch |  35 +++
 src/patches/squid/squid-3.5-14069.patch |  30 +++
 4 files changed, 449 insertions(+)
 create mode 100644 src/patches/squid/squid-3.5-14067.patch
 create mode 100644 src/patches/squid/squid-3.5-14068.patch
 create mode 100644 src/patches/squid/squid-3.5-14069.patch

diff --git a/lfs/squid b/lfs/squid
index 5bc976e..a88f0c5 100644
--- a/lfs/squid
+++ b/lfs/squid
@@ -70,6 +70,9 @@ $(subst %,%_MD5,$(objects)) :
 $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
 	@$(PREBUILD)
 	@rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE)
+	cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14067.patch
+	cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14068.patch
+	cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-3.5-14069.patch
 	cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid-3.5.17-fix-max-file-descriptors.patch
 
 	cd $(DIR_APP) && autoreconf -vfi
diff --git a/src/patches/squid/squid-3.5-14067.patch b/src/patches/squid/squid-3.5-14067.patch
new file mode 100644
index 0000000..8d9cb21
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14067.patch
@@ -0,0 +1,381 @@
+------------------------------------------------------------
+revno: 14067
+revision-id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+parent: squid3(a)treenet.co.nz-20160701113616-vpjak1pq4uecadd2
+fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
+committer: Amos Jeffries <squid3(a)treenet.co.nz>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 19:16:20 +1200
+message:
+  Bug 4534: assertion failure in xcalloc when using many cache_dir
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
+# timestamp: 2016-07-23 07:24:01 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160701113616-\
+#   vpjak1pq4uecadd2
+# 
+# Begin patch
+=== modified file 'src/CacheDigest.cc'
+--- src/CacheDigest.cc	2016-01-01 00:14:27 +0000
++++ src/CacheDigest.cc	2016-07-23 07:16:20 +0000
+@@ -35,12 +35,12 @@
+ static uint32_t hashed_keys[4];
+ 
+ static void
+-cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
++cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
+ {
+-    const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
++    const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
+     assert(cd);
+     assert(capacity > 0 && bpe > 0);
+-    assert(mask_size > 0);
++    assert(mask_size != 0);
+     cd->capacity = capacity;
+     cd->bits_per_entry = bpe;
+     cd->mask_size = mask_size;
+@@ -50,7 +50,7 @@
+ }
+ 
+ CacheDigest *
+-cacheDigestCreate(int capacity, int bpe)
++cacheDigestCreate(uint64_t capacity, uint8_t bpe)
+ {
+     CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
+     assert(SQUID_MD5_DIGEST_LENGTH == 16);  /* our hash functions rely on 16 byte keys */
+@@ -97,7 +97,7 @@
+ 
+ /* changes mask size, resets bits to 0, preserves "cd" pointer */
+ void
+-cacheDigestChangeCap(CacheDigest * cd, int new_cap)
++cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
+ {
+     assert(cd);
+     cacheDigestClean(cd);
+@@ -278,12 +278,12 @@
+     storeAppendPrintf(e, "%s digest: size: %d bytes\n",
+                       label ? label : "", stats.bit_count / 8
+                      );
+-    storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
++    storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64 " util: %d%%\n",
+                       cd->count,
+                       cd->capacity,
+                       xpercentInt(cd->count, cd->capacity)
+                      );
+-    storeAppendPrintf(e, "\t deletion attempts: %d\n",
++    storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
+                       cd->del_count
+                      );
+     storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: %d%%\n",
+@@ -297,16 +297,18 @@
+                      );
+ }
+ 
+-size_t
+-cacheDigestCalcMaskSize(int cap, int bpe)
++uint32_t
++cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
+ {
+-    return (size_t) (cap * bpe + 7) / 8;
++    uint64_t bitCount = (cap * bpe) + 7;
++    assert(bitCount < INT_MAX); // dont 31-bit overflow later
++    return static_cast<uint32_t>(bitCount / 8);
+ }
+ 
+ static void
+ cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
+ {
+-    const unsigned int bit_count = cd->mask_size * 8;
++    const uint32_t bit_count = cd->mask_size * 8;
+     unsigned int tmp_keys[4];
+     /* we must memcpy to ensure alignment */
+     memcpy(tmp_keys, key, sizeof(tmp_keys));
+
+=== modified file 'src/CacheDigest.h'
+--- src/CacheDigest.h	2016-01-01 00:14:27 +0000
++++ src/CacheDigest.h	2016-07-23 07:16:20 +0000
+@@ -22,23 +22,23 @@
+ {
+ public:
+     /* public, read-only */
+-    char *mask;         /* bit mask */
+-    int mask_size;      /* mask size in bytes */
+-    int capacity;       /* expected maximum for .count, not a hard limit */
+-    int bits_per_entry;     /* number of bits allocated for each entry from capacity */
+-    int count;          /* number of digested entries */
+-    int del_count;      /* number of deletions performed so far */
++    uint64_t count;          /* number of digested entries */
++    uint64_t del_count;      /* number of deletions performed so far */
++    uint64_t capacity;       /* expected maximum for .count, not a hard limit */
++    char *mask;              /* bit mask */
++    uint32_t mask_size;      /* mask size in bytes */
++    int8_t bits_per_entry;   /* number of bits allocated for each entry from capacity */
+ };
+ 
+-CacheDigest *cacheDigestCreate(int capacity, int bpe);
++CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
+ void cacheDigestDestroy(CacheDigest * cd);
+ CacheDigest *cacheDigestClone(const CacheDigest * cd);
+ void cacheDigestClear(CacheDigest * cd);
+-void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
++void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
+ int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
+ void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
+ void cacheDigestDel(CacheDigest * cd, const cache_key * key);
+-size_t cacheDigestCalcMaskSize(int cap, int bpe);
++uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
+ int cacheDigestBitUtil(const CacheDigest * cd);
+ void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hit, int guess_hit);
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats, StoreEntry * sentry, const char *label);
+
+=== modified file 'src/PeerDigest.h'
+--- src/PeerDigest.h	2016-01-01 00:14:27 +0000
++++ src/PeerDigest.h	2016-07-23 07:16:20 +0000
+@@ -52,7 +52,7 @@
+     store_client *old_sc;
+     HttpRequest *request;
+     int offset;
+-    int mask_offset;
++    uint32_t mask_offset;
+     time_t start_time;
+     time_t resp_time;
+     time_t expires;
+
+=== modified file 'src/peer_digest.cc'
+--- src/peer_digest.cc	2016-01-01 00:14:27 +0000
++++ src/peer_digest.cc	2016-07-23 07:16:20 +0000
+@@ -754,7 +754,7 @@
+     if (!reason && !size) {
+         if (!pd->cd)
+             reason = "null digest?!";
+-        else if (fetch->mask_offset != (int)pd->cd->mask_size)
++        else if (fetch->mask_offset != pd->cd->mask_size)
+             reason = "premature end of digest?!";
+         else if (!peerDigestUseful(pd))
+             reason = "useless digest";
+
+=== modified file 'src/store_digest.cc'
+--- src/store_digest.cc	2016-01-01 00:14:27 +0000
++++ src/store_digest.cc	2016-07-23 07:16:20 +0000
+@@ -76,36 +76,63 @@
+ static void storeDigestRewriteFinish(StoreEntry * e);
+ static EVH storeDigestSwapOutStep;
+ static void storeDigestCBlockSwapOut(StoreEntry * e);
+-static int storeDigestCalcCap(void);
+-static int storeDigestResize(void);
+ static void storeDigestAdd(const StoreEntry *);
+ 
++/// calculates digest capacity
++static uint64_t
++storeDigestCalcCap()
++{
++    /*
++     * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
++     * the bits are off). However, we do not have a formula to calculate the
++     * number of _entries_ we want to pre-allocate for.
++     */
++    const uint64_t hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
++    const uint64_t lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
++    const uint64_t e_count = StoreEntry::inUseCount();
++    uint64_t cap = e_count ? e_count : hi_cap;
++    debugs(71, 2, "have: " << e_count << ", want " << cap <<
++           " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
++
++    if (cap < lo_cap)
++        cap = lo_cap;
++
++    /* do not enforce hi_cap limit, average-based estimation may be wrong
++     *if (cap > hi_cap)
++     *  cap = hi_cap;
++     */
++
++    // Bug 4534: we still have to set an upper-limit at some reasonable value though.
++    // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
++    const uint64_t absolute_max = (INT_MAX -8) / Config.digest.bits_per_entry;
++    if (cap > absolute_max) {
++        static time_t last_loud = 0;
++        if (last_loud < squid_curtime - 86400) {
++            debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
++            last_loud = squid_curtime;
++        } else {
++            debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
++        }
++        cap = absolute_max;
++    }
++
++    return cap;
++}
+ #endif /* USE_CACHE_DIGESTS */
+ 
+-static void
+-storeDigestRegisterWithCacheManager(void)
++void
++storeDigestInit(void)
+ {
+     Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1);
+-}
+-
+-/*
+- * PUBLIC FUNCTIONS
+- */
+-
+-void
+-storeDigestInit(void)
+-{
+-    storeDigestRegisterWithCacheManager();
+ 
+ #if USE_CACHE_DIGESTS
+-    const int cap = storeDigestCalcCap();
+-
+     if (!Config.onoff.digest_generation) {
+         store_digest = NULL;
+         debugs(71, 3, "Local cache digest generation disabled");
+         return;
+     }
+ 
++    const uint64_t cap = storeDigestCalcCap();
+     store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
+     debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " <<
+            (int) Config.digest.rebuild_period << "/" <<
+@@ -290,6 +317,31 @@
+     storeDigestRebuildResume();
+ }
+ 
++/// \returns true if we actually resized the digest
++static bool
++storeDigestResize()
++{
++    const uint64_t cap = storeDigestCalcCap();
++    assert(store_digest);
++    uint64_t diff;
++    if (cap > store_digest->capacity)
++        diff = cap - store_digest->capacity;
++    else
++        diff = store_digest->capacity - cap;
++    debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
++           diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
++    /* avoid minor adjustments */
++
++    if (diff <= store_digest->capacity / 10) {
++        debugs(71, 2, "small change, will not resize.");
++        return false;
++    } else {
++        debugs(71, 2, "big change, resizing.");
++        cacheDigestChangeCap(store_digest, cap);
++    }
++    return true;
++}
++
+ /* called be Rewrite to push Rebuild forward */
+ static void
+ storeDigestRebuildResume(void)
+@@ -439,7 +491,7 @@
+     assert(e);
+     /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */
+ 
+-    if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
++    if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) > store_digest->mask_size)
+         chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
+ 
+     e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
+@@ -451,7 +503,7 @@
+     sd_state.rewrite_offset += chunk_size;
+ 
+     /* are we done ? */
+-    if (sd_state.rewrite_offset >= store_digest->mask_size)
++    if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest->mask_size)
+         storeDigestRewriteFinish(e);
+     else
+         eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false);
+@@ -467,60 +519,10 @@
+     sd_state.cblock.count = htonl(store_digest->count);
+     sd_state.cblock.del_count = htonl(store_digest->del_count);
+     sd_state.cblock.mask_size = htonl(store_digest->mask_size);
+-    sd_state.cblock.bits_per_entry = (unsigned char)
+-                                     Config.digest.bits_per_entry;
++    sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
+     sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount;
+     e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
+ }
+ 
+-/* calculates digest capacity */
+-static int
+-storeDigestCalcCap(void)
+-{
+-    /*
+-     * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
+-     * the bits are off). However, we do not have a formula to calculate the
+-     * number of _entries_ we want to pre-allocate for.
+-     */
+-    const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
+-    const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
+-    const int e_count = StoreEntry::inUseCount();
+-    int cap = e_count ? e_count :hi_cap;
+-    debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap <<
+-           " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
+-
+-    if (cap < lo_cap)
+-        cap = lo_cap;
+-
+-    /* do not enforce hi_cap limit, average-based estimation may be wrong
+-     *if (cap > hi_cap)
+-     *  cap = hi_cap;
+-     */
+-    return cap;
+-}
+-
+-/* returns true if we actually resized the digest */
+-static int
+-storeDigestResize(void)
+-{
+-    const int cap = storeDigestCalcCap();
+-    int diff;
+-    assert(store_digest);
+-    diff = abs(cap - store_digest->capacity);
+-    debugs(71, 2, "storeDigestResize: " <<
+-           store_digest->capacity << " -> " << cap << "; change: " <<
+-           diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
+-    /* avoid minor adjustments */
+-
+-    if (diff <= store_digest->capacity / 10) {
+-        debugs(71, 2, "storeDigestResize: small change, will not resize.");
+-        return 0;
+-    } else {
+-        debugs(71, 2, "storeDigestResize: big change, resizing.");
+-        cacheDigestChangeCap(store_digest, cap);
+-        return 1;
+-    }
+-}
+-
+ #endif /* USE_CACHE_DIGESTS */
+ 
+
+=== modified file 'src/tests/stub_CacheDigest.cc'
+--- src/tests/stub_CacheDigest.cc	2016-01-01 00:14:27 +0000
++++ src/tests/stub_CacheDigest.cc	2016-07-23 07:16:20 +0000
+@@ -16,11 +16,11 @@
+ class CacheDigestGuessStats;
+ class StoreEntry;
+ 
+-CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
++CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
+ void cacheDigestDestroy(CacheDigest *) STUB
+ CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
+ void cacheDigestClear(CacheDigest * ) STUB
+-void cacheDigestChangeCap(CacheDigest *,int) STUB
++void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
+ int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
+ void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
+ void cacheDigestDel(CacheDigest *, const cache_key *) STUB
+@@ -28,5 +28,4 @@
+ void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
+ void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
+-size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
+-
++uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
+
diff --git a/src/patches/squid/squid-3.5-14068.patch b/src/patches/squid/squid-3.5-14068.patch
new file mode 100644
index 0000000..4766e00
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14068.patch
@@ -0,0 +1,35 @@
+------------------------------------------------------------
+revno: 14068
+revision-id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+parent: squid3(a)treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
+fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4542
+author: Anonymous <bigparrot(a)pirateperfection.com>
+committer: Amos Jeffries <squid3(a)treenet.co.nz>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 19:19:30 +1200
+message:
+  Bug #4542: authentication credentials IP TTL updated incorrectly
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: ee0c6aab5414532d9554ef338cce049263902fd8
+# timestamp: 2016-07-23 07:24:05 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160723071620-\
+#   1wzqpbyi1rk5w6vg
+# 
+# Begin patch
+=== modified file 'src/auth/User.cc'
+--- src/auth/User.cc	2016-01-01 00:14:27 +0000
++++ src/auth/User.cc	2016-07-23 07:19:30 +0000
+@@ -284,7 +284,7 @@
+             /* This ip has already been seen. */
+             found = 1;
+             /* update IP ttl */
+-            ipdata->ip_expiretime = squid_curtime;
++            ipdata->ip_expiretime = squid_curtime + ::Config.authenticateIpTTL;
+         } else if (ipdata->ip_expiretime <= squid_curtime) {
+             /* This IP has expired - remove from the seen list */
+             dlinkDelete(&ipdata->node, &ip_list);
+
diff --git a/src/patches/squid/squid-3.5-14069.patch b/src/patches/squid/squid-3.5-14069.patch
new file mode 100644
index 0000000..15ca37a
--- /dev/null
+++ b/src/patches/squid/squid-3.5-14069.patch
@@ -0,0 +1,30 @@
+------------------------------------------------------------
+revno: 14069
+revision-id: squidadm(a)squid-cache.org-20160723121351-iuc8hwstrqd0l1dv
+parent: squid3(a)treenet.co.nz-20160723071930-cemledcltg8pkc28
+committer: Source Maintenance <squidadm(a)squid-cache.org>
+branch nick: 3.5
+timestamp: Sat 2016-07-23 12:13:51 +0000
+message:
+  SourceFormat Enforcement
+------------------------------------------------------------
+# Bazaar merge directive format 2 (Bazaar 0.90)
+# revision_id: squidadm(a)squid-cache.org-20160723121351-\
+#   iuc8hwstrqd0l1dv
+# target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# testament_sha1: c9e37a723686ae2ee489ba7ec2e981ae153bda28
+# timestamp: 2016-07-23 12:50:56 +0000
+# source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
+# base_revision_id: squid3(a)treenet.co.nz-20160723071930-\
+#   cemledcltg8pkc28
+# 
+# Begin patch
+=== modified file 'src/tests/stub_CacheDigest.cc'
+--- src/tests/stub_CacheDigest.cc	2016-07-23 07:16:20 +0000
++++ src/tests/stub_CacheDigest.cc	2016-07-23 12:13:51 +0000
+@@ -29,3 +29,4 @@
+ void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
+ void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
+ uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
++
+
-- 
2.9.2