Re: [PATCH] squid 4.4: latest patches (01-03)

Hey,

I think it is finally time (and there is a gap in the schedule) for merge squid 4.4.

Could you rebase your branch on next and send a patchset or maybe single patch if it is only that so that I can merge this? Right now I am not sure what patches I have to take to replay this on top of next.

Best, -Michael

On 8 Dec 2018, at 17:10, Matthias Fischer matthias.fischer@ipfire.org wrote:

For details see: http://www.squid-cache.org/Versions/v4/changesets/

Best, Matthias

Signed-off-by: Matthias Fischer matthias.fischer@ipfire.org

lfs/squid | 3 + ...b_exchange_with_a_TLS_cache_peer_307.patch | 91 ++++++++++++ ...all_format_formally_to_make_dist_325.patch | 22 +++ .../03_The_handshake_logformat_code_331.patch | 132 ++++++++++++++++++ 4 files changed, 248 insertions(+) create mode 100644 src/patches/squid/01_Fix_netdb_exchange_with_a_TLS_cache_peer_307.patch create mode 100644 src/patches/squid/02_Maintenance_add_xz_tarball_format_formally_to_make_dist_325.patch create mode 100644 src/patches/squid/03_The_handshake_logformat_code_331.patch

diff --git a/lfs/squid b/lfs/squid index e5e799111..aaa2d0b96 100644 --- a/lfs/squid +++ b/lfs/squid @@ -72,6 +72,9 @@ $(subst %,%_MD5,$(objects)) : $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xaf $(DIR_DL)/$(DL_FILE)

• cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/squid/01_Fix_netdb_exchange_with_a_TLS_cache_peer_307.patch

• cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/squid/02_Maintenance_add_xz_tarball_format_formally_to_make_dist_325.patch

• cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/squid/03_The_handshake_logformat_code_331.patch cd $(DIR_APP) && patch -Np0 -i $(DIR_SRC)/src/patches/squid/squid-4.4-fix-max-file-descriptors.patch

  cd $(DIR_APP) && autoreconf -vfi

diff --git a/src/patches/squid/01_Fix_netdb_exchange_with_a_TLS_cache_peer_307.patch b/src/patches/squid/01_Fix_netdb_exchange_with_a_TLS_cache_peer_307.patch new file mode 100644 index 000000000..09f8961dc --- /dev/null +++ b/src/patches/squid/01_Fix_netdb_exchange_with_a_TLS_cache_peer_307.patch @@ -0,0 +1,91 @@ +commit bc54d7a6f7ec510a25966f2f800d3ea874657546 +Author: chi-mf 43963496+chi-mf@users.noreply.github.com +Date: 2018-10-30 04:48:40 +0000

• Fix netdb exchange with a TLS cache_peer (#307)
• Squid uses http-scheme URLs when sending netdb exchange (and possibly
• other) requests to a cache_peer. If a DIRECT path is selected for that
• cache_peer URL, then Squid sends a clear text HTTP request to that
• cache_peer. If that cache_peer expects a TLS connection, it will reject
• that request (with, e.g., error:transaction-end-before-headers),
• resulting in an HTTP 503 or 504 netdb fetch error.
• Workaround this by adding an internalRemoteUri() parameter to indicate
• whether https or http URL scheme should be used. Netdb fetches from
• CachePeer::secure peers now get an https scheme and, hence, a TLS
• connection.

+diff --git a/src/icmp/net_db.cc b/src/icmp/net_db.cc +index 0f488de..526093f 100644 +--- a/src/icmp/net_db.cc ++++ b/src/icmp/net_db.cc +@@ -1282,7 +1282,7 @@ netdbExchangeStart(void *data)

• #if USE_ICMP

• CachePeer *p = (CachePeer *)data;
  

• static const SBuf netDB("netdb");
  

+- char *uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", netDB); ++ char *uri = internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-dynamic/", netDB);

• debugs(38, 3, "Requesting '" << uri << "'");
  

• const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIcmp);
  

• HttpRequest *req = HttpRequest::FromUrl(uri, mx);
  

+diff --git a/src/internal.cc b/src/internal.cc +index 6ebc7a6..ff7b4d6 100644 +--- a/src/internal.cc ++++ b/src/internal.cc +@@ -82,7 +82,7 @@ internalStaticCheck(const SBuf &urlPath)

• • makes internal url with a given host and port (remote internal url)
• */
• char *

+-internalRemoteUri(const char *host, unsigned short port, const char *dir, const SBuf &name) ++internalRemoteUri(bool encrypt, const char *host, unsigned short port, const char *dir, const SBuf &name)

• {

• static char lc_host[SQUIDHOSTNAMELEN];
  

• assert(host && !name.isEmpty());
  

+@@ -115,7 +115,7 @@ internalRemoteUri(const char *host, unsigned short port, const char *dir, const

• static MemBuf mb;
  

• mb.reset();
  

+- mb.appendf("http://" SQUIDSBUFPH, SQUIDSBUFPRINT(tmp.authority())); ++ mb.appendf("%s://" SQUIDSBUFPH, encrypt ? "https" : "http", SQUIDSBUFPRINT(tmp.authority()));

• if (dir)
  

•     mb.append(dir, strlen(dir));
  

+@@ -132,7 +132,10 @@ internalRemoteUri(const char *host, unsigned short port, const char *dir, const

• char *
• internalLocalUri(const char *dir, const SBuf &name)
• {

+- return internalRemoteUri(getMyHostname(), ++ // XXX: getMy*() may return https_port info, but we force http URIs ++ // because we have not checked whether the callers can handle https. ++ const bool secure = false; ++ return internalRemoteUri(secure, getMyHostname(),

•                          getMyPort(), dir, name);
  

• }

+diff --git a/src/internal.h b/src/internal.h +index c91f9ac..13a43a6 100644 +--- a/src/internal.h ++++ b/src/internal.h +@@ -24,7 +24,7 @@ void internalStart(const Comm::ConnectionPointer &clientConn, HttpRequest *, Sto

• bool internalCheck(const SBuf &urlPath);
• bool internalStaticCheck(const SBuf &urlPath);
• char *internalLocalUri(const char *dir, const SBuf &name);

+-char *internalRemoteUri(const char *, unsigned short, const char *, const SBuf &); ++char *internalRemoteUri(bool, const char *, unsigned short, const char *, const SBuf &);

• const char *internalHostname(void);
• int internalHostnameIs(const char *);

+diff --git a/src/peer_digest.cc b/src/peer_digest.cc +index 36a8705..f515aaa 100644 +--- a/src/peer_digest.cc ++++ b/src/peer_digest.cc +@@ -323,7 +323,7 @@ peerDigestRequest(PeerDigest * pd)

• if (p->digest_url)
  

•     url = xstrdup(p->digest_url);
  

• else
  

+- url = xstrdup(internalRemoteUri(p->host, p->http_port, "/squid-internal-periodic/", SBuf(StoreDigestFileName))); ++ url = xstrdup(internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-periodic/", SBuf(StoreDigestFileName)));

• debugs(72, 2, url);
  

• const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initCacheDigest);
  

diff --git a/src/patches/squid/02_Maintenance_add_xz_tarball_format_formally_to_make_dist_325.patch b/src/patches/squid/02_Maintenance_add_xz_tarball_format_formally_to_make_dist_325.patch new file mode 100644 index 000000000..58ceaa034 --- /dev/null +++ b/src/patches/squid/02_Maintenance_add_xz_tarball_format_formally_to_make_dist_325.patch @@ -0,0 +1,22 @@ +commit 3c23ae8c7431344f8fc50bb5ee8f4b56d08c10a4 +Author: Amos Jeffries yadij@users.noreply.github.com +Date: 2018-11-11 04:29:58 +0000

• Maintenance: add .xz tarball format formally to make dist (#325)
• Automake can now handle generating this format itself and the
• experiments of providing it for downstream have gone well.

+diff --git a/configure.ac b/configure.ac +index 3f8af6d..f668567 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -10,7 +10,7 @@ AC_PREREQ(2.61)

• AC_CONFIG_HEADERS([include/autoconf.h])
• AC_CONFIG_AUX_DIR(cfgaux)
• AC_CONFIG_SRCDIR([src/main.cc])

+-AM_INIT_AUTOMAKE([tar-ustar nostdinc subdir-objects]) ++AM_INIT_AUTOMAKE([tar-ustar nostdinc subdir-objects dist-xz])

• AC_REVISION($Revision$)dnl
• AC_PREFIX_DEFAULT(/usr/local/squid)
• AM_MAINTAINER_MODE

diff --git a/src/patches/squid/03_The_handshake_logformat_code_331.patch b/src/patches/squid/03_The_handshake_logformat_code_331.patch new file mode 100644 index 000000000..2ce8bdc4a --- /dev/null +++ b/src/patches/squid/03_The_handshake_logformat_code_331.patch @@ -0,0 +1,132 @@ +commit 0022167d80725513d95b38aaebc90086fc0b6938 (tag: refs/tags/M-staged-PR331, refs/remotes/origin/v4) +Author: Christos Tsantilas christos@chtsanti.net +Date: 2018-11-14 15:17:06 +0000

• The %>handshake logformat code (#331)
• Logging client "handshake" bytes is useful in at least two contexts:
• • Runtime traffic bypass and bumping/splicing decisions. Identifying

•  popular clients like Skype for Business (that uses a TLS handshake but
  

•  then may not speak TLS) is critical for handling their traffic
  

•  correctly. Squid does not have enough ACLs to interrogate most TLS
  

•  handshake aspects. Adding more ACLs may still be a good idea, but
  

•  initial sketches for SfB handshakes showed rather complex
  

•  ACLs/configurations, _and_ no reasonable ACLs would be able to handle
  

•  non-TLS handshakes. An external ACL receiving the handshake is in a
  

•  much better position to analyze/fingerprint it according to custom
  

•  admin needs.
  

• • A logged handshake can be used to analyze new/unusual traffic or even

•  trigger security-related alarms.
  

• The current support is limited to cases where Squid was saving handshake
• for other reasons. With enough demand, this initial support can be
• extended to all protocols and port configurations.
• This is a Measurement Factory project.

+diff --git a/src/cf.data.pre b/src/cf.data.pre +index fa8af56..a8ca587 100644 +--- a/src/cf.data.pre ++++ b/src/cf.data.pre +@@ -4394,6 +4394,37 @@ DOC_START

• <qos	Server connection TOS/DSCP value set by Squid
  

• <nfmark Server connection netfilter mark set by Squid
  

++ >handshake Raw client handshake ++ Initial client bytes received by Squid on a newly ++ accepted TCP connection or inside a just established ++ CONNECT tunnel. Squid stops accumulating handshake ++ bytes as soon as the handshake parser succeeds or ++ fails (determining whether the client is using the ++ expected protocol). ++ ++ For HTTP clients, the handshake is the request line. ++ For TLS clients, the handshake consists of all TLS ++ records up to and including the TLS record that ++ contains the last byte of the first ClientHello ++ message. For clients using an unsupported protocol, ++ this field contains the bytes received by Squid at the ++ time of the handshake parsing failure. ++ ++ See the on_unsupported_protocol directive for more ++ information on Squid handshake traffic expectations. ++ ++ Current support is limited to these contexts: ++ - http_port connections, but only when the ++ on_unsupported_protocol directive is in use. ++ - https_port connections (and CONNECT tunnels) that ++ are subject to the ssl_bump peek or stare action. ++ ++ To protect binary handshake data, this field is always ++ base64-encoded (RFC 4648 Section 4). If logformat ++ field encoding is configured, that encoding is applied ++ on top of base64. Otherwise, the computed base64 value ++ is recorded as is. ++

• Time related format codes:

• ts	Seconds since epoch
  

+diff --git a/src/format/ByteCode.h b/src/format/ByteCode.h +index ad230bb..a6f8fd9 100644 +--- a/src/format/ByteCode.h ++++ b/src/format/ByteCode.h +@@ -46,6 +46,8 @@ typedef enum {

• LFT_CLIENT_LOCAL_TOS,
  

• LFT_CLIENT_LOCAL_NFMARK,
  

++ LFT_CLIENT_HANDSHAKE, ++

• /* client connection local squid.conf details */
  

• LFT_LOCAL_LISTENING_IP,
  

• LFT_LOCAL_LISTENING_PORT,
  

+diff --git a/src/format/Format.cc b/src/format/Format.cc +index c1e19b4..8fd6720 100644 +--- a/src/format/Format.cc ++++ b/src/format/Format.cc +@@ -8,6 +8,7 @@

• #include "squid.h"
• #include "AccessLogEntry.h"

++#include "base64.h"

• #include "client_side.h"
• #include "comm/Connection.h"
• #include "err_detail_type.h"

+@@ -547,6 +548,24 @@ Format::Format::assemble(MemBuf &mb, const AccessLogEntry::Pointer &al, int logS

•         }
  

•         break;
  

++ case LFT_CLIENT_HANDSHAKE: ++ if (al->request && al->request->clientConnectionManager.valid()) { ++ const auto &handshake = al->request->clientConnectionManager->preservedClientData; ++ if (const auto rawLength = handshake.length()) { ++ // add 1 byte to optimize the c_str() conversion below ++ char *buf = sb.rawAppendStart(base64_encode_len(rawLength) + 1); ++ ++ struct base64_encode_ctx ctx; ++ base64_encode_init(&ctx); ++ auto encLength = base64_encode_update(&ctx, buf, rawLength, reinterpret_cast<const uint8_t*>(handshake.rawContent())); ++ encLength += base64_encode_final(&ctx, buf + encLength); ++ ++ sb.rawAppendFinish(buf, encLength); ++ out = sb.c_str(); ++ } ++ } ++ break; ++

•     case LFT_TIME_SECONDS_SINCE_EPOCH:
  

•         // some platforms store time in 32-bit, some 64-bit...
  

•         outoff = static_cast<int64_t>(current_time.tv_sec);
  

+diff --git a/src/format/Token.cc b/src/format/Token.cc +index 186ade5..06c60cf 100644 +--- a/src/format/Token.cc ++++ b/src/format/Token.cc +@@ -141,6 +141,7 @@ static TokenTableEntry TokenTableMisc[] = {

• TokenTableEntry("<qos", LFT_SERVER_LOCAL_TOS),
  

• TokenTableEntry(">nfmark", LFT_CLIENT_LOCAL_NFMARK),
  

• TokenTableEntry("<nfmark", LFT_SERVER_LOCAL_NFMARK),
  

++ TokenTableEntry(">handshake", LFT_CLIENT_HANDSHAKE),

• TokenTableEntry("err_code", LFT_SQUID_ERROR ),
  

• TokenTableEntry("err_detail", LFT_SQUID_ERROR_DETAIL ),
  

• TokenTableEntry("note", LFT_NOTE ),
  

-- 2.18.0