strip Content-Length header to prevent duplicate

subprojects: bump pistache
headers: add csrf (#35 )
2025-09-16 04:46:55 -05:00 · 2025-06-19 12:11:59 +02:00 · 2025-06-15 17:21:23 +02:00 · 2025-05-13 23:26:32 +01:00 · 2025-05-11 14:08:42 +02:00 · 2025-05-10 21:24:24 +01:00
15 changed files with 371 additions and 94 deletions
@@ -15,6 +15,7 @@ AI scrapers are everywhere. This will stop them. `robots.txt` won't.
 - Support for IP-Range based rules (both ipv4 and ipv6)
 - Support for async (multithreaded) request handling
 - Minimal. The waiting page is tiny and light on network usage.
+- Support for verbose traffic logging for later inspection or statistics

 ### Planned features
 - Dynamic challenge amount (aka difficulty) based on traffic
@@ -69,5 +69,24 @@
    "async_proxy": true,

    // If enabled, specific requests that look like git HTTP(s) clones will be let through.
-    "git_host": false
+    "git_host": false,
+
+    // If enabled, fingerprinting is ignored and any IP will be able to use
+    // the cached token as long as it is still valid.
+    "ignore_fingerprinting": false,
+
+    // Traffic logging to a .csv file
+    "logging": {
+        "log_traffic": false,
+
+        // This is a sample schema with all supported fields
+        // Please keep in mind your local legal regulations, as IPs under GDPR are considered personal data.
+        "traffic_log_schema": "epoch,ip,domain,resource,useragent,action",
+
+        // Where to save the logfile. Each run will continue appending to this file. It may grow HUGE! No automatic pruning / compression is done.
+        "traffic_log_file": "./traffic.csv"
+    },
+
+    // how long the token (solved challenge) should be valid for before showing a new challenge, in seconds
+    "token_valid_for": 3600 // 1 hour
 }
@@ -212,7 +212,7 @@

        <noscript>
            <p class="text-description">
-                This website uses <a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a> and
+                This website uses&nbsp;<a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a>&nbsp;and
                requires
                javascript to
                automatically verify you are not a bot.<br /><br />
@@ -227,7 +227,7 @@

            <form action="/checkpoint/challengeNoJs" method="get">
                <div>
-                    <label class="form-label" for="name">Enter the output: </label>
+                    <label class="form-label" for="name">Enter the output </label>
                    <input class="form-input" type="text" name="solution" id="solution" required />
                </div>
                <input class="input-hidden" type="text" name="fingerprint" id="fingerprint"
@@ -249,21 +249,47 @@

            <style>
                .form-label {
-                    color: #fff;
+                    color: #d9d9d9;
                    display: block;
                    text-align: center;
-                }
-
-                .form-button {
-                    margin-left: 9.5rem;
-                    width: calc(100%-19rem);
-                    display: block;
-                    position: relative;
+                    margin-bottom: 0.5rem;
+                    font-size: 1rem;
                }

                .form-input {
-                    margin-left: 9.5rem;
-                    width: calc(100%-19rem);
+                    display: block;
+                    margin: 0 auto;
+                    width: 15rem;
+                    background-color: #161616;
+                    color: #d9d9d9;
+                    border: 1px solid #280d0e;
+                    border-radius: 0.3rem;
+                    padding: 0.5rem;
+                    font-size: 1rem;
+                    margin-bottom: 0.5rem;
+                    outline: none;
+                    transition: border-color 0.2s ease-in-out;
+                }
+
+                .form-input:focus {
+                    border-color: #692225;
+                }
+
+                .form-button {
+                    display: block;
+                    margin: 0 auto;
+                    background-color: #280d0e;
+                    color: #d9d9d9;
+                    border: none;
+                    border-radius: 0.3rem;
+                    padding: 0.5rem 1.5rem;
+                    font-size: 1rem;
+                    cursor: pointer;
+                    transition: background-color 0.2s ease-in-out;
+                }
+
+                .form-button:hover {
+                    background-color: #692225;
                }

                .code-block {
@@ -1,2 +1,2 @@
-<!DOCTYPE html><html><head><title>STOP! - Checkpoint</title></head><body style="background-color:#0e0e0e;overflow:hidden"><style>@font-face{font-family:"Noto Sans";src:url(/checkpoint/NotoSans.woff)}*{font-family:"Noto Sans"}.middle-box{display:block;position:absolute;width:30rem;height:calc(100% - 2rem);left:calc(50% - 15rem);top:1rem;background-color:#111;border-radius:.5rem;border:1px solid #280d0e}.big-icon{margin:0;padding:0;font-size:8rem;color:#fff;width:100%;text-align:center}.subtext{margin:0;padding:0;font-size:3rem;color:#d9d9d9;width:100%;text-align:center}.text-hr{margin:auto;margin-top:1rem;margin-bottom:1rem;padding:0;background-color:#444;width:50%;box-shadow:none;text-align:center;height:1px;border:none}.text-hr-small{margin:auto;margin-top:1rem;margin-bottom:1rem;padding:0;background-color:#444;width:25%;box-shadow:none;text-align:center;height:1px;border:none}.text-description{margin:0;padding:0;font-size:1rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.text-question{margin:0;margin-top:4rem;padding:0;font-size:1.4rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.bottom-bar{position:absolute;width:100%;height:auto;bottom:.4rem;left:0}.bottom-hash-text{margin:0;padding:0;font-size:1rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.bottom-credit-text{display:block;margin:0;padding:0;font-size:.5rem;color:#4b4b4b;width:98%;text-align:right;transition:ease-in-out .1s;margin-top:.2rem}.bottom-credit-text:hover,.bottom-credit-text:link:hover,.bottom-credit-text:visited:hover{color:#764343;cursor:pointer}.bottom-credit-text:link,.bottom-credit-text:visited{color:#4b4b4b}.bottom-progress{position:relative;height:3px;background-color:#222;width:80%;margin-left:10%;margin-bottom:.8rem}.bottom-progress-light{position:absolute;left:0;top:0;width:0%;height:3px;background-color:#692225;transition:ease-in-out .1s}.link,.link:active,.link:focus,.link:link,.link:visited{color:#c03940;text-decoration:none}.link:hover{text-decoration:underline}@media (pointer:none),(pointer:coarse){.big-icon{margin-top:10rem}.middle-box{width:90%;left:5%}.subtext{font-size:6rem}.text-description{font-size:3rem}.text-hr{height:3px}.bottom-hash-text{font-size:2.5rem}.bottom-credit-text{font-size:1rem}}</style><div class="middle-box"><p class="big-icon">&#128721;</p><p class="subtext" id="subtext">STOP!</p><hr class="text-hr"><noscript><p class="text-description">This website uses<a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a>and requires javascript to automatically verify you are not a bot.<br><br>You seem to be running without Javascript enabled, and thus will have to do this manually.<br><br>Paste this command into your Linux terminal and provide the output below:</p><p class="code-block">perl -MDigest::SHA=sha256_hex -e '$n="{{ tl:text challengeNonce }}";$d={{ tl:text challengeDifficulty }};for($i=0;;$i++){$h=sha256_hex($n.$i);if(substr($h,0,$d) eq "0"x$d){print "$i\n";last}}'</p><form action="/checkpoint/challengeNoJs" method="get"><div><label class="form-label" for="name">Enter the output:</label><input class="form-input" type="text" name="solution" id="solution" required></div><input class="input-hidden" type="text" name="fingerprint" id="fingerprint" value="{{ tl:text challengeFingerprint }}"> <input class="input-hidden" type="text" name="challenge" id="challenge" value="{{ tl:text challengeNonce }}"> <input class="input-hidden" type="text" name="sig" id="sig" value="{{ tl:text challengeSignature }}"> <input class="input-hidden" type="text" name="timestamp" id="timestamp" value="{{ tl:text challengeTimestamp }}"> <input class="input-hidden" type="text" name="difficulty" id="difficulty" value="{{ tl:text challengeDifficulty }}"> <input class="form-button" type="submit" value="Send"></form><p class="text-description text-warning">Never trust random scripts from the internet! Make sure you understand the perl script above. If you don't, click&nbsp<a class="link" href="/checkpoint/explainNoJs" target="_blank" style="text-decoration:underline">here</a>&nbspfor an explanation.</p><style>.form-label{color:#fff;display:block;text-align:center}.form-button{margin-left:9.5rem;width:calc(100%-19rem);display:block;position:relative}.form-input{margin-left:9.5rem;width:calc(100%-19rem)}.code-block{background-color:#161616;color:#d9d9d9;border-radius:1rem;font-family:monospace;margin:1rem;width:calc(100% - 3rem);max-width:100%;text-wrap:break-word;overflow-wrap:break-word;padding:.5rem;text-align:left;border:1px solid #280d0e}.input-hidden{visibility:hidden;position:absolute}.text-warning{color:#c03940}.require-js{display:none}</style></noscript><p class="text-description require-js">Verifying that you are not a bot. This might take a short moment.<br><br>You do not need to do anything.</p><p class="text-question require-js">Why am I seeing this?</p><hr class="text-hr-small require-js"><p class="text-description require-js">This website protects itself from AILine truncated
+<!DOCTYPE html><html><head><title>STOP! - Checkpoint</title></head><body style="background-color:#0e0e0e;overflow:hidden"><style>@font-face{font-family:"Noto Sans";src:url(/checkpoint/NotoSans.woff)}*{font-family:"Noto Sans"}.middle-box{display:block;position:absolute;width:30rem;height:calc(100% - 2rem);left:calc(50% - 15rem);top:1rem;background-color:#111;border-radius:.5rem;border:1px solid #280d0e}.big-icon{margin:0;padding:0;font-size:8rem;color:#fff;width:100%;text-align:center}.subtext{margin:0;padding:0;font-size:3rem;color:#d9d9d9;width:100%;text-align:center}.text-hr{margin:auto;margin-top:1rem;margin-bottom:1rem;padding:0;background-color:#444;width:50%;box-shadow:none;text-align:center;height:1px;border:none}.text-hr-small{margin:auto;margin-top:1rem;margin-bottom:1rem;padding:0;background-color:#444;width:25%;box-shadow:none;text-align:center;height:1px;border:none}.text-description{margin:0;padding:0;font-size:1rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.text-question{margin:0;margin-top:4rem;padding:0;font-size:1.4rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.bottom-bar{position:absolute;width:100%;height:auto;bottom:.4rem;left:0}.bottom-hash-text{margin:0;padding:0;font-size:1rem;color:#d9d9d9;width:85%;margin-left:7.5%;text-align:center}.bottom-credit-text{display:block;margin:0;padding:0;font-size:.5rem;color:#4b4b4b;width:98%;text-align:right;transition:ease-in-out .1s;margin-top:.2rem}.bottom-credit-text:hover,.bottom-credit-text:link:hover,.bottom-credit-text:visited:hover{color:#764343;cursor:pointer}.bottom-credit-text:link,.bottom-credit-text:visited{color:#4b4b4b}.bottom-progress{position:relative;height:3px;background-color:#222;width:80%;margin-left:10%;margin-bottom:.8rem}.bottom-progress-light{position:absolute;left:0;top:0;width:0%;height:3px;background-color:#692225;transition:ease-in-out .1s}.link,.link:active,.link:focus,.link:link,.link:visited{color:#c03940;text-decoration:none}.link:hover{text-decoration:underline}@media (pointer:none),(pointer:coarse){.big-icon{margin-top:10rem}.middle-box{width:90%;left:5%}.subtext{font-size:6rem}.text-description{font-size:3rem}.text-hr{height:3px}.bottom-hash-text{font-size:2.5rem}.bottom-credit-text{font-size:1rem}}</style><div class="middle-box"><p class="big-icon">&#128721;</p><p class="subtext" id="subtext">STOP!</p><hr class="text-hr"><noscript><p class="text-description">This website uses&nbsp;<a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a>&nbsp;and requires javascript to automatically verify you are not a bot.<br><br>You seem to be running without Javascript enabled, and thus will have to do this manually.<br><br>Paste this command into your Linux terminal and provide the output below:</p><p class="code-block">perl -MDigest::SHA=sha256_hex -e '$n="{{ tl:text challengeNonce }}";$d={{ tl:text challengeDifficulty }};for($i=0;;$i++){$h=sha256_hex($n.$i);if(substr($h,0,$d) eq "0"x$d){print "$i\n";last}}'</p><form action="/checkpoint/challengeNoJs" method="get"><div><label class="form-label" for="name">Enter the output</label><input class="form-input" type="text" name="solution" id="solution" required></div><input class="input-hidden" type="text" name="fingerprint" id="fingerprint" value="{{ tl:text challengeFingerprint }}"> <input class="input-hidden" type="text" name="challenge" id="challenge" value="{{ tl:text challengeNonce }}"> <input class="input-hidden" type="text" name="sig" id="sig" value="{{ tl:text challengeSignature }}"> <input class="input-hidden" type="text" name="timestamp" id="timestamp" value="{{ tl:text challengeTimestamp }}"> <input class="input-hidden" type="text" name="difficulty" id="difficulty" value="{{ tl:text challengeDifficulty }}"> <input class="form-button" type="submit" value="Send"></form><p class="text-description text-warning">Never trust random scripts from the internet! Make sure you understand the perl script above. If you don't, click&nbsp<a class="link" href="/checkpoint/explainNoJs" target="_blank" style="text-decoration:underline">here</a>&nbspfor an explanation.</p><style>.form-label{color:#d9d9d9;display:block;text-align:center;margin-bottom:.5rem;font-size:1rem}.form-input{display:block;margin:0 auto;width:15rem;background-color:#161616;color:#d9d9d9;border:1px solid #280d0e;border-radius:.3rem;padding:.5rem;font-size:1rem;margin-bottom:.5rem;outline:0;transition:border-color .2s ease-in-out}.form-input:focus{border-color:#692225}.form-button{display:block;margin:0 auto;background-color:#280d0e;color:#d9d9d9;border:none;border-radius:.3rem;padding:.5rem 1.5rem;font-size:1rem;cursor:pointer;transition:background-color .2s ease-in-out}.form-button:hover{background-color:#692225}.code-block{background-color:#161616;color:#d9d9d9;border-radius:1rem;font-family:monospace;margin:1rem;width:calc(100% - 3rem);max-width:100%;text-wrap:break-word;overflow-wrap:break-word;padding:.5rem;text-align:left;border:1px solid #280d0e}.input-hiLine truncated
 <script type="text/javascript">setTimeout(async function(){var e=0,t=Date.now();let l="{{ tl:text challengeNonce }}",n=parseInt("{{ tl:text challengeDifficulty }}");function $(e){return e>1e6?parseInt(e/1e5)/10+"M":parseInt(e/100)/10+"k"}function r(e){return e>1e3?parseInt(e/100)/10+"s":e+"ms"}function o(e){return Math.floor((1-1/(Math.pow(e/Math.pow(2,4*n)*3,2)+1))*100)}function i(e){document.getElementById("results").innerHTML="Success! Completed challenge after "+$(e)+" iterations, in "+r(Math.floor(Date.now()-t))+".",document.getElementsByName("progress-light")[0].style.width="100%";let o=JSON.stringify({challenge:l,solution:e,difficulty:n,sig:"{{ tl:text challengeSignature }}",fingerprint:"{{ tl:text challengeFingerprint }}",timestamp:"{{ tl:text challengeTimestamp }}"});fetch("/checkpoint/challenge",{headers:{"Content-Type":"application/json"},method:"POST",body:o}).then(e=>{200==e.status?(console.log("Got token."),window.location.reload()):console.log("Server error")})}let s=new Uint8Array(l.length+10);for(let a=0;a<l.length;a++)s[a]=l.charCodeAt(a);for(;;){let c=0===e?1:~~Math.floor(Math.log10(e))+1;for(let g=c-1;g>=0;g--)s[l.length+(c-1-g)]=48+~~Math.floor(e/Math.pow(10,g)%10);let _=new Uint8Array(await window.crypto.subtle.digest("sha-256",s.slice(0,l.length+c))),f=0;for(;f<n&&(f%2==0?_[~~(f/2)]>>4:15&_[~~(f/2)])==0;f++);if(f>=n){i(e),console.log("Success: it "+e+": "+Array.from(_).map(e=>e.toString(16).padStart(2,"0")).join(""));break}if(++e%11377==0){let h=Math.floor(Date.now()-t),u=e/(h/1e3);document.getElementById("results").innerHTML="Difficulty: "+n+", elapsed "+r(h)+", "+$(e)+"h, "+$(u)+"h/s",document.getElementsByName("progress-light")[0].style.width=o(e)+"%"}}},100);var currentTitle=1;setInterval(()=>{let e=["STOP","HALT","ST&#0211;J","ARR&#0202;T","&#1057;&#1058;&#1030;&#1049;"];document.getElementById("subtext").innerHTML=e[currentTitle]+"!",++currentTitle>=e.length&&(currentTitle=0)},2e3);</script></body></html>
@@ -35,8 +35,16 @@ class CConfig {
        bool                     trace_logging                = false;
        std::vector<SConfigRule> rules                        = {};
        int                      default_challenge_difficulty = 4;
+        int                      token_valid_for              = 3600;
+        bool                     ignore_fingerprinting        = false;
        bool                     async_proxy                  = true;
        std::vector<SProxyRule>  proxy_rules;
+
+        struct {
+            bool        log_traffic = false;
+            std::string traffic_log_schema;
+            std::string traffic_log_file;
+        } logging;
    } m_config;

    struct {
@@ -49,12 +49,7 @@ CChallenge::CChallenge(const std::string& jsonResponse) {
 CChallenge::CChallenge(const Pistache::Http::Request& reqResponse) {
    auto& q = reqResponse.query();

-    if (!q.has("solution")
-        || !q.has("fingerprint")
-        || !q.has("challenge")
-        || !q.has("timestamp")
-        || !q.has("sig")
-        || !q.has("difficulty"))
+    if (!q.has("solution") || !q.has("fingerprint") || !q.has("challenge") || !q.has("timestamp") || !q.has("sig") || !q.has("difficulty"))
        return;

    m_challenge   = q.get("challenge").value();
@@ -14,6 +14,8 @@
 #include "../GlobalState.hpp"
 #include "../config/Config.hpp"
 #include "../helpers/FsUtils.hpp"
+#include "../helpers/RequestUtils.hpp"
+#include "../logging/TrafficLogger.hpp"

 #include <filesystem>
 #include <random>
@@ -25,7 +27,6 @@
 #include <openssl/evp.h>
 #include <magic.h>

-constexpr const uint64_t TOKEN_MAX_AGE_MS  = 1000 * 60 * 60; // 1hr
 constexpr const char* TOKEN_COOKIE_NAME = "checkpoint-token";

 //
@@ -56,73 +57,10 @@ static std::string generateToken() {
    return ss.str();
 }

-std::string CServerHandler::fingerprintForRequest(const Pistache::Http::Request& req) {
-    const auto                                                    HEADERS = req.headers();
-    std::shared_ptr<const Pistache::Http::Header::AcceptEncoding> acceptEncodingHeader;
-    std::shared_ptr<const Pistache::Http::Header::UserAgent>      userAgentHeader;
-    std::shared_ptr<const AcceptLanguageHeader>                   languageHeader;
-
-    std::string                                                   input = "checkpoint-";
-
-    try {
-        acceptEncodingHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::AcceptEncoding>(HEADERS.get("Accept-Encoding"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        languageHeader = Pistache::Http::Header::header_cast<AcceptLanguageHeader>(HEADERS.get("Accept-Language"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        userAgentHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(HEADERS.get("User-Agent"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    input += ipForRequest(req);
-    // TODO: those seem to change. Find better things to hash.
-    // if (acceptEncodingHeader)
-    //     input += HEADERS.getRaw("Accept-Encoding").value();
-    // if (languageHeader)
-    //     input += languageHeader->language();
-    if (userAgentHeader)
-        input += userAgentHeader->agent();
-
-    return g_pCrypto->sha256(input);
-}
-
 bool CServerHandler::isResourceCheckpoint(const std::string_view& res) {
    return res.starts_with("/checkpoint/");
 }

-std::string CServerHandler::ipForRequest(const Pistache::Http::Request& req) {
-    std::shared_ptr<const CFConnectingIPHeader> cfHeader;
-    std::shared_ptr<const XRealIPHeader>        xRealIPHeader;
-
-    try {
-        cfHeader = Pistache::Http::Header::header_cast<CFConnectingIPHeader>(req.headers().get("cf-connecting-ip"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        xRealIPHeader = Pistache::Http::Header::header_cast<XRealIPHeader>(req.headers().get("X-Real-IP"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    if (cfHeader)
-        return cfHeader->ip();
-
-    if (xRealIPHeader)
-        return xRealIPHeader->ip();
-
-    return req.address().host();
-}
-
 void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter response) {
    const auto                                                 HEADERS = req.headers();
    std::shared_ptr<const Pistache::Http::Header::Host>        hostHeader;
@@ -186,7 +124,7 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt

    Debug::log(LOG, "New request: {}:{}{}", hostHeader->host(), hostHeader->port().toString(), req.resource());

-    const auto REQUEST_IP = ipForRequest(req);
+    const auto REQUEST_IP = NRequestUtils::ipForRequest(req);

    Debug::log(LOG, " | Request author: IP {}, direct: {}", REQUEST_IP, req.address().host());

@@ -228,12 +166,14 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
                Debug::log(TRACE, "Request looks like it is coming from git (UA + GP). Accepting.");

                proxyPass(req, response);
+                g_pTrafficLogger->logTraffic(req, "PASS (git)");
                return;
            } else if (userAgentHeader->agent().starts_with("git/")) {
                Debug::log(LOG, " | Action: PASS (git)");
                Debug::log(TRACE, "Request looks like it is coming from git (UA git). Accepting.");

                proxyPass(req, response);
+                g_pTrafficLogger->logTraffic(req, "PASS (git)");
                return;
            }
        }
@@ -249,10 +189,12 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
                    case IP_ACTION_DENY:
                        Debug::log(LOG, " | Action: DENY (rule)");
                        response.send(Pistache::Http::Code::Forbidden, "Blocked by checkpoint");
+                        g_pTrafficLogger->logTraffic(req, "DENY (rule)");
                        return;
                    case IP_ACTION_ALLOW:
                        Debug::log(LOG, " | Action: PASS (rule)");
                        proxyPass(req, response);
+                        g_pTrafficLogger->logTraffic(req, "PASS (rule)");
                        return;
                    case IP_ACTION_CHALLENGE:
                        Debug::log(LOG, " | Action: CHALLENGE (rule)");
@@ -273,12 +215,13 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
        if (TOKEN.valid()) {
            const auto AGE = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count() -
                std::chrono::duration_cast<std::chrono::milliseconds>(TOKEN.issued().time_since_epoch()).count();
-            if (AGE <= TOKEN_MAX_AGE_MS && TOKEN.fingerprint() == fingerprintForRequest(req)) {
+            if (AGE <= g_pConfig->m_config.token_valid_for * 1000 && (TOKEN.fingerprint() == NRequestUtils::fingerprintForRequest(req) || g_pConfig->m_config.ignore_fingerprinting)) {
                Debug::log(LOG, " | Action: PASS (token)");
+                g_pTrafficLogger->logTraffic(req, "PASS (token)");
                proxyPass(req, response);
                return;
            } else { // token has been used from a different IP or is expired. Nuke it.
-                if (AGE > TOKEN_MAX_AGE_MS)
+                if (AGE > g_pConfig->m_config.token_valid_for * 1000)
                    Debug::log(LOG, " | Action: CHALLENGE (token expired)");
                else
                    Debug::log(LOG, " | Action: CHALLENGE (token fingerprint mismatch)");
@@ -305,12 +248,14 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
        if (ec) {
            // bad resource
            response.send(Pistache::Http::Code::Bad_Request, "Bad Request");
+            g_pTrafficLogger->logTraffic(req, "BAD_CHECKPOINT_RESOURCE");
            return;
        }

        if (!PATH_ABSOLUTE.string().starts_with(HTML_ROOT)) {
            // directory traversal
            response.send(Pistache::Http::Code::Bad_Request, "Bad Request");
+            g_pTrafficLogger->logTraffic(req, "BAD_CHECKPOINT_RESOURCE");
            return;
        }

@@ -326,9 +271,12 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt

        auto body = NFsUtils::readFileAsString(PATH_ABSOLUTE).value_or("");
        response.send(body.empty() ? Pistache::Http::Code::Internal_Server_Error : Pistache::Http::Code::Ok, body);
+        g_pTrafficLogger->logTraffic(req, "PASS (Checkpoint resource)");
        return;
    }

+    g_pTrafficLogger->logTraffic(req, "CHALLENGE");
+
    serveStop(req, response, challengeDifficulty);
 }

@@ -338,7 +286,7 @@ void CServerHandler::onTimeout(const Pistache::Http::Request& request, Pistache:

 void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool js) {
    const auto JSON        = req.body();
-    const auto FINGERPRINT = fingerprintForRequest(req);
+    const auto FINGERPRINT = NRequestUtils::fingerprintForRequest(req);

    CChallenge CHALLENGE;
    if (!js)
@@ -348,6 +296,7 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist

    if (!CHALLENGE.valid()) {
        response.send(Pistache::Http::Code::Bad_Request, "Bad request");
+        g_pTrafficLogger->logTraffic(req, "CHALLENGE_FAIL");
        return;
    }

@@ -368,7 +317,7 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist
    }

    response.headers().add(
-        std::make_shared<SetCookieHeader>(std::string{TOKEN_COOKIE_NAME} + "=" + TOKEN.tokenCookie() + "; Domain=" + hostDomain + "; HttpOnly; Path=/; Secure; SameSite=Lax"));
+        std::make_shared<SetCookieHeader>(std::string{TOKEN_COOKIE_NAME} + "=" + TOKEN.tokenCookie() + "; Domain=" + hostDomain + "; Max-Age=" + std::to_string(g_pConfig->m_config.token_valid_for) + "; HttpOnly; Path=/; Secure; SameSite=Lax"));

    if (js)
        response.send(Pistache::Http::Code::Ok, "Ok");
@@ -376,6 +325,8 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist
        response.headers().add<Pistache::Http::Header::Location>("/");
        response.send(Pistache::Http::Code::Moved_Permanently, "");
    }
+
+    g_pTrafficLogger->logTraffic(req, "CHALLENGE_PASS");
 }

 void CServerHandler::serveStop(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, int difficulty) {
@@ -385,7 +336,7 @@ void CServerHandler::serveStop(const Pistache::Http::Request& req, Pistache::Htt
    page.setTemplateRoot(PAGE_ROOT);

    const auto NONCE     = generateNonce();
-    const auto CHALLENGE = CChallenge(fingerprintForRequest(req), NONCE, difficulty);
+    const auto CHALLENGE = CChallenge(NRequestUtils::fingerprintForRequest(req), NONCE, difficulty);

    auto       hostDomain = req.headers().getRaw("Host").value();
    if (hostDomain.contains(":"))
@@ -496,7 +447,7 @@ void CServerHandler::proxyPassInternal(const Pistache::Http::Request& req, Pista
            const auto HEADERSRESP = resp.headers().list();

            for (auto& h : HEADERSRESP) {
-                if (std::string_view{h->name()} == "Transfer-Encoding") {
+                if (std::string_view{h->name()} == "Transfer-Encoding" || std::string_view{h->name()} == "Content-Length") {
                    Debug::log(TRACE, "Header out: {}: {} (DROPPED)", h->name(), resp.headers().getRaw(h->name()).value());
                    continue;
                }
@@ -22,8 +22,6 @@ class CServerHandler : public Pistache::Http::Handler {
    void proxyPassInternal(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool async = false);
    void proxyPassAsync(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response);
    void challengeSubmitted(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool js);
-    std::string fingerprintForRequest(const Pistache::Http::Request& req);
-    std::string ipForRequest(const Pistache::Http::Request& req);

    bool isResourceCheckpoint(const std::string_view& res);

@@ -0,0 +1,24 @@
+#include <pistache/http_headers.h>
+#include <pistache/net.h>
+
+class XCSRFTokenHeader : public Pistache::Http::Header::Header {
+  public:
+    NAME("X-Csrf-Token");
+
+    XCSRFTokenHeader() = default;
+
+    void parse(const std::string& str) override {
+        m_data = str;
+    }
+
+    void write(std::ostream& os) const override {
+        os << m_data;
+    }
+
+    std::string ip() const {
+        return m_data;
+    }
+
+  private:
+    std::string m_data = "";
+};
@@ -0,0 +1,75 @@
+#include "RequestUtils.hpp"
+
+#include "../core/Crypto.hpp"
+
+#include "../headers/authorization.hpp"
+#include "../headers/cfHeader.hpp"
+#include "../headers/xforwardfor.hpp"
+#include "../headers/gitProtocolHeader.hpp"
+#include "../headers/wwwAuthenticateHeader.hpp"
+#include "../headers/acceptLanguageHeader.hpp"
+#include "../headers/setCookieHeader.hpp"
+#include "../headers/xrealip.hpp"
+
+std::string NRequestUtils::fingerprintForRequest(const Pistache::Http::Request& req) {
+    const auto                                                    HEADERS = req.headers();
+    std::shared_ptr<const Pistache::Http::Header::AcceptEncoding> acceptEncodingHeader;
+    std::shared_ptr<const Pistache::Http::Header::UserAgent>      userAgentHeader;
+    std::shared_ptr<const AcceptLanguageHeader>                   languageHeader;
+
+    std::string                                                   input = "checkpoint-";
+
+    try {
+        acceptEncodingHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::AcceptEncoding>(HEADERS.get("Accept-Encoding"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        languageHeader = Pistache::Http::Header::header_cast<AcceptLanguageHeader>(HEADERS.get("Accept-Language"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        userAgentHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(HEADERS.get("User-Agent"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    input += ipForRequest(req);
+    // TODO: those seem to change. Find better things to hash.
+    // if (acceptEncodingHeader)
+    //     input += HEADERS.getRaw("Accept-Encoding").value();
+    // if (languageHeader)
+    //     input += languageHeader->language();
+    if (userAgentHeader)
+        input += userAgentHeader->agent();
+
+    return g_pCrypto->sha256(input);
+}
+
+std::string NRequestUtils::ipForRequest(const Pistache::Http::Request& req) {
+    std::shared_ptr<const CFConnectingIPHeader> cfHeader;
+    std::shared_ptr<const XRealIPHeader>        xRealIPHeader;
+
+    try {
+        cfHeader = Pistache::Http::Header::header_cast<CFConnectingIPHeader>(req.headers().get("cf-connecting-ip"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        xRealIPHeader = Pistache::Http::Header::header_cast<XRealIPHeader>(req.headers().get("X-Real-IP"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    if (cfHeader)
+        return cfHeader->ip();
+
+    if (xRealIPHeader)
+        return xRealIPHeader->ip();
+
+    return req.address().host();
+}
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <string>
+
+#include <pistache/http.h>
+
+namespace NRequestUtils {
+    std::string fingerprintForRequest(const Pistache::Http::Request& req);
+    std::string ipForRequest(const Pistache::Http::Request& req);
+};
@@ -0,0 +1,132 @@
+#include "TrafficLogger.hpp"
+
+#include <sstream>
+#include <fmt/format.h>
+
+#include "../config/Config.hpp"
+#include "../debug/log.hpp"
+#include "../helpers/RequestUtils.hpp"
+
+CTrafficLogger::CTrafficLogger() {
+    if (!g_pConfig->m_config.logging.log_traffic)
+        return;
+
+    const auto COMMAS = std::count(g_pConfig->m_config.logging.traffic_log_schema.begin(), g_pConfig->m_config.logging.traffic_log_schema.end(), ',');
+
+    // parse the schema
+    std::string_view curr;
+    size_t           lastPos = 0;
+    bool             first   = true;
+    auto             advance = [&]() {
+        size_t prev = !first ? lastPos + 1 : lastPos;
+        lastPos     = g_pConfig->m_config.logging.traffic_log_schema.find(',', prev);
+
+        if (lastPos == std::string::npos)
+            curr = std::string_view{g_pConfig->m_config.logging.traffic_log_schema}.substr(prev);
+        else
+            curr = std::string_view{g_pConfig->m_config.logging.traffic_log_schema}.substr(prev, lastPos - prev);
+
+        first = false;
+    };
+
+    for (size_t i = 0; i < COMMAS + 1; ++i) {
+        advance();
+
+        if (curr == "ip")
+            m_logSchema.emplace_back(TRAFFIC_IP);
+        else if (curr == "epoch")
+            m_logSchema.emplace_back(TRAFFIC_EPOCH);
+        else if (curr == "domain")
+            m_logSchema.emplace_back(TRAFFIC_DOMAIN);
+        else if (curr == "resource")
+            m_logSchema.emplace_back(TRAFFIC_RESOURCE);
+        else if (curr == "useragent")
+            m_logSchema.emplace_back(TRAFFIC_USERAGENT);
+        else if (curr == "action")
+            m_logSchema.emplace_back(TRAFFIC_ACTION);
+
+        if (curr == "")
+            break;
+    }
+
+    m_file.open(g_pConfig->m_config.logging.traffic_log_file, std::ios::app);
+
+    if (!m_file.good())
+        Debug::die("TrafficLogger: bad file {}", g_pConfig->m_config.logging.traffic_log_file);
+}
+
+CTrafficLogger::~CTrafficLogger() {
+    if (m_file.is_open())
+        m_file.close();
+}
+
+static std::string sanitize(const std::string& s) {
+    if (s.empty())
+        return s;
+
+    std::string cpy = s;
+    size_t      pos = 0;
+    while ((pos = cpy.find('"', pos)) != std::string::npos) {
+        cpy.replace(pos, 1, "\\\"");
+        pos += 2;
+    }
+
+    return cpy;
+}
+
+void CTrafficLogger::logTraffic(const Pistache::Http::Request& req, const char* actionTaken) {
+    if (!g_pConfig->m_config.logging.log_traffic)
+        return;
+
+    std::stringstream ss;
+
+    for (const auto& t : m_logSchema) {
+        switch (t) {
+            case TRAFFIC_EPOCH: {
+                ss << fmt::format("{},", std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count());
+                break;
+            }
+
+            case TRAFFIC_DOMAIN: {
+                const auto HOST = Pistache::Http::Header::header_cast<Pistache::Http::Header::Host>(req.headers().get("Host"));
+                ss << fmt::format("\"{}\",", sanitize(HOST->host()));
+                break;
+            }
+
+            case TRAFFIC_IP: {
+                ss << fmt::format("{},", NRequestUtils::ipForRequest(req));
+                break;
+            }
+
+            case TRAFFIC_RESOURCE: {
+                ss << fmt::format("\"{}\",", sanitize(req.resource()));
+                break;
+            }
+
+            case TRAFFIC_USERAGENT: {
+                if (!req.headers().has("User-Agent")) {
+                    ss << "\"<no data>\",";
+                    break;
+                }
+                const auto UA = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(req.headers().get("User-Agent"));
+                ss << fmt::format("\"{}\",", sanitize(UA->agent()));
+                break;
+            }
+
+            case TRAFFIC_ACTION: {
+                ss << fmt::format("{},", actionTaken);
+                break;
+            }
+        }
+    }
+
+    std::string trafficLine = ss.str();
+    if (trafficLine.empty())
+        return;
+
+    // replace , with \n
+    trafficLine.back() = '\n';
+
+    m_file << trafficLine;
+    m_file.flush();
+}
@@ -0,0 +1,33 @@
+#pragma once
+
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <fstream>
+
+#include "../config/ConfigTypes.hpp"
+
+#include <pistache/http.h>
+
+class CTrafficLogger {
+  public:
+    CTrafficLogger();
+    ~CTrafficLogger();
+
+    void logTraffic(const Pistache::Http::Request& req, const char* actionTaken);
+
+  private:
+    enum eTrafficLoggerProps : uint8_t {
+        TRAFFIC_EPOCH = 0,
+        TRAFFIC_IP,
+        TRAFFIC_DOMAIN,
+        TRAFFIC_RESOURCE,
+        TRAFFIC_USERAGENT,
+        TRAFFIC_ACTION,
+    };
+
+    std::vector<eTrafficLoggerProps> m_logSchema;
+    std::ofstream                    m_file;
+};
+
+inline std::unique_ptr<CTrafficLogger> g_pTrafficLogger;
@@ -16,6 +16,7 @@
 #include "headers/acceptLanguageHeader.hpp"
 #include "headers/setCookieHeader.hpp"
 #include "headers/xrealip.hpp"
+#include "headers/csrfHeader.hpp"

 #include "debug/log.hpp"
 #include "helpers/FsUtils.hpp"
@@ -25,6 +26,8 @@

 #include "config/Config.hpp"

+#include "logging/TrafficLogger.hpp"
+
 #include "GlobalState.hpp"

 #include <signal.h>
@@ -86,8 +89,10 @@ int main(int argc, char** argv, char** envp) {
    Pistache::Http::Header::Registry::instance().registerHeader<AcceptLanguageHeader>();
    Pistache::Http::Header::Registry::instance().registerHeader<SetCookieHeader>();
    Pistache::Http::Header::Registry::instance().registerHeader<XRealIPHeader>();
+    Pistache::Http::Header::Registry::instance().registerHeader<XCSRFTokenHeader>();

    g_pCrypto        = std::make_unique<CCrypto>();
+    g_pTrafficLogger = std::make_unique<CTrafficLogger>();

    auto endpoint = std::make_unique<Pistache::Http::Endpoint>(address);
    auto opts     = Pistache::Http::Endpoint::options().threads(threads).flags(Pistache::Tcp::Options::ReuseAddr | Pistache::Tcp::Options::ReusePort);
Author	SHA1	Message	Date
Kouya Heika	e6df2d2a2d	strip Content-Length header to prevent duplicate	2025-09-16 04:46:55 -05:00
Vaxry	929d1ee80a	subprojects: bump pistache	2025-06-19 12:11:59 +02:00
Vaxry	7c3bb4cbd9	headers: add csrf (#35 )	2025-06-15 17:21:23 +02:00
catfromplan9	b606a21912	handler: Add ignore_fingerprinting to config (#33 )	2025-05-13 23:26:32 +01:00
catfromplan9	02313315af	handler: Add Max-Age to cookies (#32 ) Add Max-Age to checkpoint-token cookie, so that the cookie persists after the end of the current session.	2025-05-11 14:08:42 +02:00
Vaxry	5022e37917	handler: fix valid for taking ms	2025-05-10 21:24:24 +01:00
Vaxry	5afa967338	config: add token_valid_for	2025-05-10 19:50:22 +01:00
Vaxry	467d19d8c3	logging: improve action	2025-04-28 17:43:57 +01:00
Vaxry	fd9baeeab8	traffic: Add a new traffic logger	2025-04-28 17:29:28 +01:00
nyxandVaxry	119da9c973	html: minor fixes to styling (#28 ) * e * minify --------- Co-authored-by: Vaxry <vaxry@vaxry.net>	2025-04-24 00:05:16 +02:00