From 1d86a3a73050f646c17b75501809dbc1c31211a3 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Tue, 25 Feb 2020 18:43:59 +0530 Subject: [PATCH 01/27] block page new interop api --- goproxy/adblock.go | 145 +++++------ goproxy/adblock_import.go | 89 ++++--- goproxy/adblock_interop.go | 93 ++++---- goproxy/certs.go | 95 +++++++- goproxy/debug.go | 19 -- goproxy/main.go | 477 ++++--------------------------------- goproxy/response.go | 2 +- 7 files changed, 318 insertions(+), 602 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index b27bbb5..6100810 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -5,10 +5,13 @@ import ( "encoding/gob" "log" "os" - "regexp" + "runtime/debug" "strings" "github.com/patriciy/adblock/adblock" + + goahocorasick "github.com/anknown/ahocorasick" + lru "github.com/hashicorp/golang-lru" ) const ( @@ -17,22 +20,27 @@ const ( ) const MAX_RULES_PER_MATCHER = 1000 -const MAX_CONTENT_SIZE_SCAN = 200 * 1024 //200kb max to scan -var adblockMatcher *AdBlockMatcher +const MAX_CONTENT_SIZE_SCAN = 1000 * 1024 //500kb max to scan +var adBlockMatcher *AdBlockMatcher var defaultBlockPageContent = "%url% is blocked. Category %category%. Reason %reason%" +var lruCache, _ = lru.New(1024) + +type cacheItem struct { + category *string + matchType int +} type MatcherCategory struct { - CategoryId int32 - ListType int32 + Category string Matchers []*adblock.RuleMatcher BlockedDomains map[string]bool } type PhraseCategory struct { - Category string - Phrases []string - regexp *regexp.Regexp + Category string + Phrases []string + processor *goahocorasick.Machine } type AdBlockMatcher struct { @@ -48,19 +56,19 @@ type AdBlockMatcher struct { } func CreateMatcher() *AdBlockMatcher { - adblockMatcher = &AdBlockMatcher{ + adBlockMatcher = &AdBlockMatcher{ RulesCnt: 0, BlockPageContent: defaultBlockPageContent, } - return adblockMatcher + return adBlockMatcher } -func (am *AdBlockMatcher) addMatcher(categoryId int32, listType int32, bypass bool) { +func (am *AdBlockMatcher) addMatcher(category string, bypass bool) { matcher := adblock.NewMatcher() var categoryMatcher *MatcherCategory - for _, element := range adblockMatcher.MatcherCategories { - if element.CategoryId == categoryId { + for _, element := range adBlockMatcher.MatcherCategories { + if element.Category == category { categoryMatcher = element break } @@ -68,8 +76,7 @@ func (am *AdBlockMatcher) addMatcher(categoryId int32, listType int32, bypass bo if categoryMatcher == nil { categoryMatcher = &MatcherCategory{ - CategoryId: categoryId, - ListType: listType, + Category: category, BlockedDomains: make(map[string]bool), } @@ -81,8 +88,8 @@ func (am *AdBlockMatcher) addMatcher(categoryId int32, listType int32, bypass bo } categoryMatcher.Matchers = append(categoryMatcher.Matchers, matcher) - adblockMatcher.lastMatcher = matcher - adblockMatcher.lastCategory = categoryMatcher + adBlockMatcher.lastMatcher = matcher + adBlockMatcher.lastCategory = categoryMatcher } func (am *AdBlockMatcher) GetBlockPage(url string, category string, reason string) string { @@ -92,72 +99,64 @@ func (am *AdBlockMatcher) GetBlockPage(url string, category string, reason strin return tagsReplacer.Replace(am.BlockPageContent) } -func (am *AdBlockMatcher) TestUrlBlockedWithMatcherCategories(url string, host string, referer string) ([]*MatcherCategory, []int) { +func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) (*string, int) { + if am.RulesCnt == 0 { + return nil, Included + } + + cacheKey := url + host + if v, ok := lruCache.Get(cacheKey); ok { + item := v.(cacheItem) + + log.Printf("Cache hit: %s %d", url, item.matchType) + return item.category, item.matchType + } res1, res2 := am.matchRulesCategories(am.MatcherCategories, url, host, referer) - if len(res1) > 0 { + if res1 != nil { + lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2}) return res1, res2 } if am.bypassEnabled { - return make([]*MatcherCategory, 0), make([]int, 0) + return nil, Included } - return am.matchRulesCategories(am.BypassMatcherCategories, url, host, referer) -} - -func TransformMatcherCategoryArrayToIntArray(categories []*MatcherCategory) []int32 { - ret := make([]int32, len(categories)) - - for i, category := range categories { - ret[i] = category.CategoryId + res1, res2 = am.matchRulesCategories(am.BypassMatcherCategories, url, host, referer) + if res1 != nil { + lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2}) } - return ret -} - -func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) []int32 { - categories, _ := am.TestUrlBlockedWithMatcherCategories(url, host, referer) - return TransformMatcherCategoryArrayToIntArray(categories) + lruCache.Add(cacheKey, cacheItem{category: nil, matchType: Included}) + return res1, res2 } -func (am *AdBlockMatcher) matchRulesCategories(matcherCategories []*MatcherCategory, url string, host string, referer string) ([]*MatcherCategory, []int) { +func (am *AdBlockMatcher) matchRulesCategories(matcherCategories []*MatcherCategory, url string, host string, referer string) (*string, int) { rq := &adblock.Request{ URL: url, Domain: host, Referer: referer, } - var matchedCategories []*MatcherCategory - var catergoriesMatchType []int //Included, Excluded - domainParts := strings.Split(host, ".") - for _, matcherCategory := range matcherCategories { - categoryMatched := false for _, matcher := range matcherCategory.Matchers { - matched, categoryType, err := matcher.Match(rq) + matched, matchType, err := matcher.Match(rq) if err != nil { log.Printf("Error matching rule %s", err) } if matched { - categoryMatched = true - matchedCategories = append(matchedCategories, matcherCategory) - catergoriesMatchType = append(catergoriesMatchType, categoryType) - break + return &matcherCategory.Category, matchType } } - if !categoryMatched { - matched, matchType := matchDomain(domainParts, matcherCategory) - if matched { - matchedCategories = append(matchedCategories, matcherCategory) - catergoriesMatchType = append(catergoriesMatchType, matchType) - } + matched, matchType := matchDomain(domainParts, matcherCategory) + if matched { + return &matcherCategory.Category, matchType } } - return matchedCategories, catergoriesMatchType + return nil, Included } func matchDomain(domainParts []string, matcherCatergory *MatcherCategory) (bool, int) { @@ -188,15 +187,20 @@ func (am *AdBlockMatcher) TestContentTypeIsFiltrable(contentType string) bool { } func (am *AdBlockMatcher) IsContentSmallEnoughToFilter(contentSize int64) bool { - return contentSize < MAX_CONTENT_SIZE_SCAN + return contentSize > 0 && contentSize < MAX_CONTENT_SIZE_SCAN } func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) *string { + text := []rune(strings.ToLower(string(str))) + for _, phraseCategory := range am.PhraseCategories { - if phraseCategory.regexp != nil { - if phraseCategory.regexp.Find(str) != nil { - return &phraseCategory.Category - } + if phraseCategory.processor == nil { + continue + } + + res := phraseCategory.processor.MultiPatternSearch(text, true) + if len(res) > 0 { + return &phraseCategory.Category } } @@ -205,7 +209,7 @@ func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) *string { func (am *AdBlockMatcher) AddBlockedPhrase(phrase string, category string) { var phraseCategory *PhraseCategory = nil - for _, element := range adblockMatcher.PhraseCategories { + for _, element := range adBlockMatcher.PhraseCategories { if element.Category == category { phraseCategory = element break @@ -220,24 +224,31 @@ func (am *AdBlockMatcher) AddBlockedPhrase(phrase string, category string) { am.PhraseCategories = append(am.PhraseCategories, phraseCategory) } - phraseCategory.Phrases = append(phraseCategory.Phrases, regexp.QuoteMeta(phrase)) + phraseCategory.Phrases = append(phraseCategory.Phrases, phrase) } func (am *AdBlockMatcher) Build() { am.phrasesCount = 0 for _, phraseCategory := range am.PhraseCategories { - regexString := strings.Join(phraseCategory.Phrases, "|") + processor := new(goahocorasick.Machine) - var e error - phraseCategory.regexp, e = regexp.Compile("(?i)" + regexString) - if e != nil { - log.Printf("Error compiling matcher %s", e) + dict := [][]rune{} + for _, phrase := range phraseCategory.Phrases { + dict = append(dict, []rune(strings.ToLower(phrase))) } + processor.Build(dict) + phraseCategory.processor = processor + am.phrasesCount += len(phraseCategory.Phrases) } + if len(am.MatcherCategories) == 0 { + return + } matchers := am.MatcherCategories[len(am.MatcherCategories)-1].Matchers am.lastMatcher = matchers[len(matchers)-1] + + debug.FreeOSMemory() } func (am *AdBlockMatcher) RulesCount() int { @@ -283,14 +294,14 @@ func LoadMatcherFromFile(filePath string) *AdBlockMatcher { decoder := gob.NewDecoder(stream) - adblockMatcher = &AdBlockMatcher{ + adBlockMatcher = &AdBlockMatcher{ RulesCnt: 0, } - err = decoder.Decode(&adblockMatcher) + err = decoder.Decode(&adBlockMatcher) if err != nil { log.Printf("Decoder error %s", err) } - return adblockMatcher + return adBlockMatcher } func (am *AdBlockMatcher) EnableBypass() { diff --git a/goproxy/adblock_import.go b/goproxy/adblock_import.go index 75c313b..464049b 100644 --- a/goproxy/adblock_import.go +++ b/goproxy/adblock_import.go @@ -1,22 +1,28 @@ package main import ( + "archive/zip" "bufio" + "io/ioutil" "log" - "os" + "strings" "github.com/patriciy/adblock/adblock" ) -const ( - Blacklist = 1 - Whitelist = 2 - BypassList = 3 -) - -func (am *AdBlockMatcher) AddRule(rule string, categoryId int32, listType int32) { - bypass := listType == BypassList +func (am *AdBlockMatcher) ParseRulesZipArchive(filePath string) { + zipFile, e := zip.OpenReader(filePath) + if e != nil { + log.Printf("Error parsing zipfile %s", e) + return + } + defer zipFile.Close() + for _, file := range zipFile.File { + am.ParseZipRulesFile(file) + } +} +func (am *AdBlockMatcher) AddRule(rule string, category string, bypass bool) { r, e := adblock.ParseRule(rule) if e != nil { @@ -28,8 +34,8 @@ func (am *AdBlockMatcher) AddRule(rule string, categoryId int32, listType int32) return } - if am.RulesCnt > 0 && am.RulesCnt%MAX_RULES_PER_MATCHER == 0 { - am.addMatcher(categoryId, listType, bypass) + if am.RulesCnt%MAX_RULES_PER_MATCHER == 0 { + am.addMatcher(category, bypass) } //Check if it's just a domain rule @@ -42,42 +48,67 @@ func (am *AdBlockMatcher) AddRule(rule string, categoryId int32, listType int32) } } } + am.lastMatcher.AddRule(r, am.RulesCnt) - r = nil + am.RulesCnt = am.RulesCnt + 1 } -func (am *AdBlockMatcher) ParseRuleFile(fileName string, categoryId int32, listType int32) { - file, err := os.Open(fileName) - defer file.Close() +func (am *AdBlockMatcher) ParseZipRulesFile(file *zip.File) { + fileDescriptor, err := file.Open() + defer fileDescriptor.Close() if err != nil { - log.Printf("Error opening rule file %s with error %s", fileName, err) + log.Printf("Error open zip file %s", err) return } - scanner := bufio.NewScanner(file) + if strings.Contains(file.Name, "block.htm") { + am.addBlockPageFromZipFile(file) + } else { + scanner := bufio.NewScanner(fileDescriptor) + categoryName := file.Name + if strings.Contains(file.Name, ".triggers") { + log.Printf("Opening triggers %s", file.Name) + am.addPhrasesFromScanner(scanner, categoryName) + } else if strings.Contains(file.Name, ".bypass") { + am.addMatcher(categoryName, true) + log.Printf("Opening bypass %s", file.Name) + am.addRulesFromScanner(scanner, categoryName, true) + } else if strings.Contains(file.Name, ".rules") { + am.addMatcher(categoryName, false) + log.Printf("Opening rules %s", file.Name) + am.addRulesFromScanner(scanner, categoryName, false) + } else { + log.Printf("File type recognition failed %s", file.Name) + } + } - bypass := listType == BypassList +} - am.addMatcher(categoryId, listType, bypass) - log.Printf("Opening rules %s", fileName) - am.addRulesFromScanner(scanner, categoryId, listType) +func (am *AdBlockMatcher) addBlockPageFromZipFile(file *zip.File) { + fileReader, e := file.Open() + if e != nil { + log.Printf("Error reading block page %s %s", e, file.Name) + } + defer fileReader.Close() + content, e := ioutil.ReadAll(fileReader) + if e != nil { + log.Printf("Error reading block page %s %s", e, file.Name) + } + am.BlockPageContent = string(content) } -func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryId int32, listType int32) { +func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, bypass bool) { for scanner.Scan() { line := scanner.Text() - - am.AddRule(line, categoryId, listType) + am.AddRule(line, categoryName, bypass) } - - adblock.ClearCaches() } -/*func (am *AdBlockMatcher) addPhrasesFromScanner(scanner *bufio.Scanner, categoryId int32) { +func (am *AdBlockMatcher) addPhrasesFromScanner(scanner *bufio.Scanner, categoryName string) { for scanner.Scan() { line := scanner.Text() - am.AddBlockedPhrase(line, categoryId) + am.AddBlockedPhrase(line, categoryName) } -}*/ +} diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 4724a80..3515f56 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -4,20 +4,23 @@ import "C" import ( "bufio" + "io/ioutil" "log" - "net/http" - "net/textproto" - "strings" + "os" "unsafe" ) -var adBlockMatcher *AdBlockMatcher = nil - -var onWhitelistCallback unsafe.Pointer -var onBlacklistCallback unsafe.Pointer +var adBlockBlacklistCallback unsafe.Pointer var adBlockMatchers map[int32]*AdBlockMatcher +const ( + Blacklist = 1 + Whitelist = 2 + BypassList = 3 + TextTrigger = 4 +) + //export AdBlockMatcherInitialize func AdBlockMatcherInitialize() { var oldMatcher *AdBlockMatcher = nil @@ -34,59 +37,51 @@ func AdBlockMatcherInitialize() { } //export AdBlockMatcherParseRuleFile -func AdBlockMatcherParseRuleFile(fileName string, categoryId int32, listType int32) { - log.Printf("AdBlockMatcherParseRuleFile11(%s, %d, %d)", fileName, categoryId, listType) - adBlockMatcher.ParseRuleFile(fileName, categoryId, listType) -} - -//export AdBlockMatcherSave -func AdBlockMatcherSave(fileName string) { - adBlockMatcher.SaveToFile(fileName) -} - -//export AdBlockMatcherLoad -func AdBlockMatcherLoad(fileName string) { - adBlockMatcher = LoadMatcherFromFile(fileName) -} - -//export AdBlockMatcherTestUrlMatch -func AdBlockMatcherTestUrlMatch(url string, host string, headersRaw string) []int32 { - var headers http.Header = nil +func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listType int32) bool { + fileName := C.GoString(fileNameC) + categoryId := C.GoString(categoryIdC) - if len(headersRaw) > 0 { - reader := bufio.NewReader(strings.NewReader(headersRaw + "\r\n")) - tp := textproto.NewReader(reader) + fileHandle, err := os.Open(fileName) + if err != nil { + return false + } + defer fileHandle.Close() - mimeHeader, err := tp.ReadMIMEHeader() - if err != nil { - log.Printf("MIME Header parse error: %s", err) - } + scanner := bufio.NewScanner(fileHandle) - headers = http.Header(mimeHeader) + if listType == TextTrigger { + adBlockMatcher.addPhrasesFromScanner(scanner, categoryId) + } else { + adBlockMatcher.addRulesFromScanner(scanner, categoryId, listType == BypassList) } - - return adBlockMatcher.TestUrlBlocked(url, host, headers.Get("referer")) + return true } -//export AdBlockMatcherAreListsLoaded -func AdBlockMatcherAreListsLoaded() bool { - if adBlockMatcher == nil { - return false - } else if adBlockMatcher.MatcherCategories == nil && adBlockMatcher.BypassMatcherCategories == nil { - return false - } else { - return len(adBlockMatcher.MatcherCategories) > 0 || len(adBlockMatcher.BypassMatcherCategories) > 0 +//export AdBlockMatcherSetBlockedPageContent +func AdBlockMatcherSetBlockedPageContent(contentC *C.char) { + blockPagePath := C.GoString(contentC) + fileHandle, err := os.Open(blockPagePath) + if err != nil { + log.Printf("Error reading block page %s", err) + return } + defer fileHandle.Close() + content, e := ioutil.ReadAll(fileHandle) + if e != nil { + log.Printf("Error reading block page %s", e) + return + } + adBlockMatcher.BlockPageContent = string(content) } -//export AdBlockMatcherSetWhitelistCallback -func AdBlockMatcherSetWhitelistCallback(callback unsafe.Pointer) { - onWhitelistCallback = callback +//export AdBlockMatcherSave +func AdBlockMatcherSave(fileName string) { + adBlockMatcher.SaveToFile(fileName) } -//export AdBlockMatcherSetBlacklistCallback -func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { - onBlacklistCallback = callback +//export AdBlockMatcherLoad +func AdBlockMatcherLoad(fileName string) { + adBlockMatcher = LoadMatcherFromFile(fileName) } //export AdBlockMatcherEnableBypass diff --git a/goproxy/certs.go b/goproxy/certs.go index 0f15adc..864e02e 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -1,12 +1,22 @@ package main import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" "io/ioutil" "log" + "math/big" + "os" + "time" - "github.com/cloudveiltech/goproxy" + "github.com/elazarl/goproxy" ) var defaultTLSConfig = &tls.Config{ @@ -97,3 +107,86 @@ func verifyCerts(dnsName string, peerCerts []*x509.Certificate) (bool, error) { return true, nil } + +func publicKey(priv interface{}) interface{} { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + default: + return nil + } +} + +func pemBlockForKey(priv interface{}) *pem.Block { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)} + case *ecdsa.PrivateKey: + b, err := x509.MarshalECPrivateKey(k) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err) + os.Exit(2) + } + return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} + default: + return nil + } +} + +func GenerateCerts(caCertPath, caKeyPath string) bool { + + // priv, err := rsa.GenerateKey(rand.Reader, *rsaBits) + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + log.Printf("Error generating cert %s", err) + return false + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Cloudveil Filtering Certificate"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 3650), + + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv) + if err != nil { + log.Printf("Failed to create certificate: %s", err) + return false + } + + certFile, err := os.Create(caCertPath) + if err != nil { + log.Printf("Error generating cert %s", err) + return false + } + defer certFile.Close() + pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + + certKeyFile, err := os.Create(caKeyPath) + if err != nil { + log.Printf("Unable to marshal ECDSA private key: %v", err) + return false + } + defer certKeyFile.Close() + keyBytes, err := x509.MarshalECPrivateKey(priv) + if err != nil { + log.Printf("Unable to marshal ECDSA private key: %v", err) + return false + } + if err := pem.Encode(certKeyFile, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}); err != nil { + log.Printf("Unable to marshal ECDSA private key: %v", err) + return false + } + + return true +} diff --git a/goproxy/debug.go b/goproxy/debug.go index 70f615c..d2ab2b9 100644 --- a/goproxy/debug.go +++ b/goproxy/debug.go @@ -6,9 +6,7 @@ import ( "net/http" "os" "runtime" - "runtime/debug" "strings" - "time" ) //import _ "net/http/pprof" @@ -17,23 +15,6 @@ func d(msg string) { fmt.Fprint(os.Stderr, msg) } -func monitorMemoryUsage() { - go func() { - for { - time.Sleep(30 * time.Second) - printMemUsage() - debug.FreeOSMemory() - if !IsRunning() { - return - } - } - }() - - // go func() { - // http.ListenAndServe(":6060", nil) - // }() -} - func printMemUsage() { var m runtime.MemStats runtime.ReadMemStats(&m) diff --git a/goproxy/main.go b/goproxy/main.go index 34608dd..fd64896 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -1,73 +1,37 @@ package main -/* -typedef int (*callback)(long long id); -typedef int (*adBlockCallback)(long long id, _GoString_ url, int* categories, int categoryLen); - -static inline int FireCallback(void *ptr, long long id) -{ - callback p = (callback)ptr; - return p(id); -} - -static inline int FireAdblockCallback(void* ptr, long long id, _GoString_ url, int* categories, int categoryLen) -{ - adBlockCallback p = (adBlockCallback)ptr; - return p(id, url, categories, categoryLen); -} - -*/ import "C" import ( - "bufio" - "bytes" - "context" - "crypto/tls" "fmt" - "io" "log" "net" - "net/http" - "net/url" "os" - "strings" - "time" + "runtime/debug" "unsafe" - - "github.com/cloudveiltech/goproxy" - "github.com/inconshreveable/go-vhost" ) -type Config struct { - portHttp int16 - portHttps int16 -} - -var ( - proxy *goproxy.ProxyHttpServer - server *http.Server - config = Config{8080, 8081} - - beforeRequestCallback unsafe.Pointer - beforeResponseCallback unsafe.Pointer +const ( + SUCCESS = 1 + ERROR_PORTS_BUSY = -1 + ERROR_CERTS_GENERATION = -2 ) -const proxyNextActionKey string = "__proxyNextAction__" - -//export SetOnBeforeRequestCallback -func SetOnBeforeRequestCallback(callback unsafe.Pointer) { - beforeRequestCallback = callback -} +func checkPortAvailable(port int16) bool { + l, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + defer l.Close() -//export SetOnBeforeResponseCallback -func SetOnBeforeResponseCallback(callback unsafe.Pointer) { - beforeResponseCallback = callback + if err != nil { + // Log or report the error here + return false + } + return true } //export SetProxyLogFile -func SetProxyLogFile(logFile string) { - file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) +func SetProxyLogFile(logFile *C.char) { + logPath := C.GoString(logFile) + file, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return } @@ -75,376 +39,37 @@ func SetProxyLogFile(logFile string) { redirectStderr(file) } -//export Init -func Init(portHttp int16, portHttps int16, certFile string, keyFile string) { - goproxy.SetDefaultTlsConfig(defaultTLSConfig) - loadAndSetCa(certFile, keyFile) - proxy = goproxy.NewProxyHttpServer() - proxy.Verbose = true - - if proxy.Verbose { - log.Printf("certFilePath %s", certFile) - } - - proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - log.Printf("NonproxyHandler fired.") - - if req.Host == "" { - fmt.Fprintln(w, "Cannot handle requests without Host header, e.g., HTTP 1.0") - return - } - - req.URL.Scheme = "http" - req.URL.Host = req.Host - proxy.ServeHTTP(w, req) - }) - - proxy.WebSocketHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - h, ok := w.(http.Hijacker) - if !ok { - return - } - - client, _, err := h.Hijack() - if err != nil { - log.Printf("Websocket error Hijack %s", err) - return - } - - remote := dialRemote(req) - if remote == nil { - return - } - - defer remote.Close() - defer client.Close() - - log.Printf("Got websocket request %s %s", req.Host, req.URL) - - req.Write(remote) - go func() { - for { - n, err := io.Copy(remote, client) - if err != nil { - log.Printf("Websocket error request %s", err) - return - } - if n == 0 { - log.Printf("Websocket nothing requested close") - return - } - time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop - } - }() - - for { - n, err := io.Copy(client, remote) - if err != nil { - log.Printf("Websocket error response %s", err) - return - } - if n == 0 { - log.Printf("Websocket nothing responded close") - return - } - time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop - } - }) - - proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) - config.portHttp = portHttp - config.portHttps = portHttps - - if proxy.Verbose { - log.Printf("Server inited") - } +//export AdBlockMatcherSetBlacklistCallback +func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { + adBlockBlacklistCallback = callback } -func dialRemote(req *http.Request) net.Conn { +//export StartGoServer +func StartGoServer(portHttp int16, portHttps int16, certFileC *C.char, keyFileC *C.char) int16 { + debug.SetTraceback("all") + debug.SetPanicOnFault(true) - port := "" - if !strings.Contains(req.Host, ":") { - if req.URL.Scheme == "https" { - port = ":443" - } else { - port = ":80" - } + if !checkPortAvailable(portHttp) || !checkPortAvailable(portHttps) { + return ERROR_PORTS_BUSY } - log.Printf("Custom dialer %s", req.Host+port) + certFile := C.GoString(certFileC) + keyFile := C.GoString(keyFileC) - if req.URL.Scheme == "https" { - conf := tls.Config{ - InsecureSkipVerify: true, - Renegotiation: tls.RenegotiateFreelyAsClient, - } - remote, err := tls.Dial("tcp", req.Host+port, &conf) - if err != nil { - log.Printf("Websocket error connect %s", err) - return nil + _, err := os.Stat(certFile) + if os.IsNotExist(err) { + if !GenerateCerts(certFile, keyFile) { + return ERROR_CERTS_GENERATION } - return remote - } else { - remote, err := net.Dial("tcp", req.Host+port) - if err != nil { - log.Printf("Websocket error connect %s", err) - return nil - } - return remote } -} -func startHttpServer() *http.Server { - srv := &http.Server{Addr: fmt.Sprintf(":%d", config.portHttp)} - srv.Handler = proxy - - go func() { - err := srv.ListenAndServe() - if err != nil { - // cannot panic, because this probably is an intentional close - log.Printf("Httpserver: ListenAndServe() error: %s", err) - server = nil - } - }() - - // returning reference so caller can call Shutdown() - return srv + startGoProxyServer(portHttp, portHttps, certFile, keyFile) + return SUCCESS } -//export Start -func Start() { - if proxy == nil { - return - } - - if proxy.Verbose { - log.Printf("Server is about to start") - } - - server = startHttpServer() - - proxy.OnRequest().DoFunc( - func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { - userData := make(map[string]interface{}) - ctx.UserData = userData - userData["blocked"] = false - - //dumpRequest(r) - request := r - var response *http.Response = nil - - session := session{r, nil, false} - id := saveSessionToInteropMap(ctx.Session, &session) - defer removeSessionFromInteropMap(id) - - if beforeRequestCallback != nil { - blocked := int32(C.FireCallback(beforeRequestCallback, C.longlong(id))) == 1 - userData["blocked"] = blocked - - request = session.request - response = session.response - } - - if response != nil { - return request, response - } - - // Now run our matching engine. - if AdBlockMatcherAreListsLoaded() { - - url := request.URL.String() - host := request.URL.Hostname() - - // adBlockMatcher is in adblock_interop.go - categories, matchTypes := adBlockMatcher.TestUrlBlockedWithMatcherCategories(url, host, request.Referer()) - if len(categories) > 0 { - for index, category := range categories { - if category.ListType == Whitelist || matchTypes[index] == Excluded { - userData["blocked"] = false - - - log.Printf("Whitelisted categories matched %s", request.URL.String()) - if onWhitelistCallback != nil { - categoryInts := TransformMatcherCategoryArrayToIntArray(categories) - - C.FireAdblockCallback(onWhitelistCallback, C.longlong(id), url, (*C.int)(&categoryInts[0]), C.int(len(categoryInts))) - - request = session.request - } - - return request, nil - } - } - - if categories[0].ListType == Blacklist || categories[0].ListType == BypassList { - userData["blocked"] = true - - log.Printf("Blacklisted categories matched %s", request.URL.String()) - if onBlacklistCallback != nil { - categoryInts := TransformMatcherCategoryArrayToIntArray(categories) - - C.FireAdblockCallback(onBlacklistCallback, C.longlong(id), url, (*C.int)(&categoryInts[0]), C.int(len(categoryInts))) - - request = session.request - response = session.response - } - - return request, response //goproxy.NewResponse(request, "text/plain", 401, "Blocked by rules") - } - } else { - log.Printf("No categories matched %s", request.URL.String()) - } - } else { - log.Printf("No categories loaded %s", request.URL.String()) - } - - return request, response - }) - - proxy.OnResponse().DoFunc( - func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { - response := resp - var isVerified bool = true - - if response != nil && response.TLS != nil { - var err error - isVerified, err = verifyCerts(ctx.Req.URL.Host, response.TLS.PeerCertificates) - if err != nil { - isVerified = false - } - } else { - isVerified = false - } - - if ctx.UserData != nil { - userData, ok := ctx.UserData.(map[string]interface{}) - - if ok { - blocked, valueOk := userData["blocked"].(bool) - if valueOk { - if !blocked { - return response - } - } - } - } - - // TODO: Call x509.Certificate.Verify - // We should be able to glean from that whether or not we do bad SSL page. - // A couple of things here: - // 1. Need a boolean that says IsVerified for Response - // 2. Need a block page that allows us to bypass it directly from the block page. - if beforeResponseCallback != nil { - session := session{ctx.Req, resp, isVerified} - session.isCertVerified = isVerified - id := saveSessionToInteropMap(ctx.Session, &session) - C.FireCallback(beforeResponseCallback, C.longlong(id)) - removeSessionFromInteropMap(id) - - response = session.response - - //log.Printf("OnBeforeResponse overhead time: %v, %v", time.Since(startTime), id) - } - - return response - }) - - go runHttpsListener() - - if proxy.Verbose { - log.Printf("Server started %d, %d", config.portHttp, config.portHttps) - } - - monitorMemoryUsage() -} - -func runHttpsListener() { - log.Printf("runHttpsListener() %d", config.portHttps) - - // listen to the TLS ClientHello but make it a CONNECT request instead - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", config.portHttps)) - - if err != nil { - log.Fatalf("Error listening for https connections - %v", err) - return - } - - for { - c, err := ln.Accept() - if err != nil { - log.Printf("Error accepting new connection - %v", err) - continue - } - - go func(c net.Conn) { - tlsConn, err := vhost.TLS(c) - if err != nil { - log.Printf("Error accepting new connection - %v", err) - } - if tlsConn.Host() == "" { - log.Printf("Cannot support non-SNI enabled clients") - return - } - - if proxy.Verbose { - log.Printf("Https handler called for %s", tlsConn.Host()) - } - - connectReq := &http.Request{ - Method: "CONNECT", - URL: &url.URL{ - Opaque: tlsConn.Host(), - Host: net.JoinHostPort(tlsConn.Host(), "443"), - }, - Host: tlsConn.Host(), - Header: make(http.Header), - } - - resp := dumbResponseWriter{tlsConn} - proxy.ServeHTTP(resp, connectReq) - }(c) - } -} - -type dumbResponseWriter struct { - net.Conn -} - -func (dumb dumbResponseWriter) Header() http.Header { - // panic("Header() should not be called on this ResponseWriter") - return make(http.Header) -} - -func (dumb dumbResponseWriter) Write(buf []byte) (int, error) { - if bytes.Equal(buf, []byte("HTTP/1.0 200 OK\r\n\r\n")) { - return len(buf), nil // throw away the HTTP OK response from the faux CONNECT request - } - return dumb.Conn.Write(buf) -} - -func (dumb dumbResponseWriter) WriteHeader(code int) { - // panic("WriteHeader() should not be called on this ResponseWriter") -} - -func (dumb dumbResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { - return dumb, bufio.NewReadWriter(bufio.NewReader(dumb), bufio.NewWriter(dumb)), nil -} - -//export Stop -func Stop() { - context, _ := context.WithTimeout(context.Background(), 1*time.Second) - server.Shutdown(context) - server = nil -} - -//export IsRunning -func IsRunning() bool { - return server != nil -} - -//export GetCert -func GetCert(res *[]byte) { - *res = caCert +//export StopGoServer +func StopGoServer() { + stopGoProxyServer() } func main() { @@ -453,37 +78,17 @@ func main() { func test() { log.Printf("main: starting HTTP server") -/* - AdBlockMatcherInitialize() - adBlockMatcher.ParseRuleFile("c:/Users/dgora/Downloads/class_33/whitelist.rules", 0, Blacklist) - categories, matchTypes := adBlockMatcher.TestUrlBlockedWithMatcherCategories("https://mapbox.com/", "mapbox.com", "") - if len(categories) > 0 { - for index := range categories { - if matchTypes[index] == Excluded { - log.Print("Matched Excluded") - } else { - log.Print("Matched Included") - } - } - - } else { - log.Print("not matched") - }*/ - Init(14500, 14501, "rootCertificate.pem", "rootPrivateKey.pem") - Start() + startGoProxyServer(14500, 14501, "rootCertificate.pem", "rootPrivateKey.pem") log.Printf("main: serving for 1000 seconds") var quit = false - var line = "" - - reader := bufio.NewReader(os.Stdin) for !quit { - line, _ = reader.ReadString('\n') - if strings.TrimSpace(line) == "quit" { - quit = true - } + //line, _ = reader.ReadString('\n') + //if strings.TrimSpace(line) == "quit" { + // quit = true + //} } // Stop() diff --git a/goproxy/response.go b/goproxy/response.go index 58d10cc..7ccae77 100644 --- a/goproxy/response.go +++ b/goproxy/response.go @@ -8,7 +8,7 @@ import ( "io/ioutil" "strings" - "github.com/cloudveiltech/goproxy" + "github.com/elazarl/goproxy" ) //export ResponseGetStatusCode From 3cc1efd932141db26e6c201b66473fddfb7dd7cf Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Wed, 26 Feb 2020 16:22:38 +0530 Subject: [PATCH 02/27] fix whitelist --- goproxy/adblock_import.go | 9 ++++++--- goproxy/adblock_interop.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/goproxy/adblock_import.go b/goproxy/adblock_import.go index 464049b..76cc84d 100644 --- a/goproxy/adblock_import.go +++ b/goproxy/adblock_import.go @@ -74,11 +74,11 @@ func (am *AdBlockMatcher) ParseZipRulesFile(file *zip.File) { } else if strings.Contains(file.Name, ".bypass") { am.addMatcher(categoryName, true) log.Printf("Opening bypass %s", file.Name) - am.addRulesFromScanner(scanner, categoryName, true) + am.addRulesFromScanner(scanner, categoryName, false, true) } else if strings.Contains(file.Name, ".rules") { am.addMatcher(categoryName, false) log.Printf("Opening rules %s", file.Name) - am.addRulesFromScanner(scanner, categoryName, false) + am.addRulesFromScanner(scanner, categoryName, false, false) } else { log.Printf("File type recognition failed %s", file.Name) } @@ -99,9 +99,12 @@ func (am *AdBlockMatcher) addBlockPageFromZipFile(file *zip.File) { am.BlockPageContent = string(content) } -func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, bypass bool) { +func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, whitelist bool, bypass bool) { for scanner.Scan() { line := scanner.Text() + if whitelist && !strings.HasPrefix(line, "@@") { + line = "@@" + line + } am.AddRule(line, categoryName, bypass) } } diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 3515f56..d8b0897 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -52,7 +52,7 @@ func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listTyp if listType == TextTrigger { adBlockMatcher.addPhrasesFromScanner(scanner, categoryId) } else { - adBlockMatcher.addRulesFromScanner(scanner, categoryId, listType == BypassList) + adBlockMatcher.addRulesFromScanner(scanner, categoryId, listType == Whitelist, listType == BypassList) } return true } From 6d9c717e58f11b495806d3a2197b34e02f92c10c Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Sat, 14 Mar 2020 13:59:45 +0700 Subject: [PATCH 03/27] bugfix --- goproxy/adblock_interop.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index d8b0897..f3137f0 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -34,6 +34,8 @@ func AdBlockMatcherInitialize() { if oldMatcher != nil { adBlockMatcher.bypassEnabled = oldMatcher.bypassEnabled } + + lruCache.Purge() } //export AdBlockMatcherParseRuleFile @@ -49,6 +51,10 @@ func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listTyp scanner := bufio.NewScanner(fileHandle) + log.Printf("Parsing category %s file %s", categoryId, fileName) + + adBlockMatcher.addMatcher(categoryId, listType == BypassList) + if listType == TextTrigger { adBlockMatcher.addPhrasesFromScanner(scanner, categoryId) } else { From cffd8c66a4d746cf7d70074a96fa49f853b49bb3 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 2 Apr 2020 16:56:48 +0700 Subject: [PATCH 04/27] add handlebars block page support --- goproxy/adblock.go | 87 +++++++++++++++++++++++++++++--------- goproxy/adblock_import.go | 4 +- goproxy/adblock_interop.go | 47 ++++++++++++++++---- 3 files changed, 109 insertions(+), 29 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 6100810..bd1e944 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -4,13 +4,17 @@ import ( "compress/gzip" "encoding/gob" "log" + "net/url" "os" "runtime/debug" "strings" "github.com/patriciy/adblock/adblock" + "encoding/base64" + goahocorasick "github.com/anknown/ahocorasick" + "github.com/aymerick/raymond" lru "github.com/hashicorp/golang-lru" ) @@ -23,12 +27,13 @@ const MAX_RULES_PER_MATCHER = 1000 const MAX_CONTENT_SIZE_SCAN = 1000 * 1024 //500kb max to scan var adBlockMatcher *AdBlockMatcher -var defaultBlockPageContent = "%url% is blocked. Category %category%. Reason %reason%" +var defaultBlockPageContent = "{{url_text}} is blocked. Category {{matching_category}}. Reason {{message}}" var lruCache, _ = lru.New(1024) type cacheItem struct { - category *string - matchType int + category *string + matchType int + isRelaxedPolicy bool } type MatcherCategory struct { @@ -52,13 +57,15 @@ type AdBlockMatcher struct { RulesCnt int phrasesCount int bypassEnabled bool - BlockPageContent string + BlockPageTemplate *raymond.Template + BlockCertTemplate *raymond.Template + defaultBlockPageTags map[string]string } func CreateMatcher() *AdBlockMatcher { adBlockMatcher = &AdBlockMatcher{ - RulesCnt: 0, - BlockPageContent: defaultBlockPageContent, + RulesCnt: 0, + defaultBlockPageTags: make(map[string]string), } return adBlockMatcher @@ -92,16 +99,56 @@ func (am *AdBlockMatcher) addMatcher(category string, bypass bool) { adBlockMatcher.lastCategory = categoryMatcher } -func (am *AdBlockMatcher) GetBlockPage(url string, category string, reason string) string { - tagsReplacer := strings.NewReplacer("%url%", url, - "%category%", category, - "%reason%", reason) - return tagsReplacer.Replace(am.BlockPageContent) +func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPolicy, isTriggerBlocked bool) string { + tags := am.defaultBlockPageTags + + tags["url_text"] = blockedUrl + tags["friendly_url_text"] = blockedUrl + tags["message"] = "" + tags["matching_category"] = category + if isRelaxedPolicy { + tags["isRelaxedPolicy"] = "1" + } else { + tags["isRelaxedPolicy"] = "" + } + + if isTriggerBlocked { + tags["showUnblockRequestButton"] = "" + } else { + tags["showUnblockRequestButton"] = "1" + } + + tags["unblockRequest"] = tags["unblockRequestBase"] + "&category_name=" + url.QueryEscape(category) + "&blocked_request=" + base64.StdEncoding.EncodeToString([]byte(blockedUrl)) + + res, err := am.BlockPageTemplate.Exec(tags) + if err != nil { + log.Printf("Error render block block page %v", err) + return "Blocked default page" + } + return res +} + +func (am *AdBlockMatcher) GetBadCertPage(url, category, host, certThumbPrint string) string { + tags := am.defaultBlockPageTags + tags["url_text"] = url + tags["friendly_url_text"] = url + tags["certThumbprintExists"] = certThumbPrint + tags["host"] = host + + if am.BlockCertTemplate == nil { + return "Blocked cert default page" + } + res, err := am.BlockCertTemplate.Exec(tags) + if err != nil { + log.Printf("Error render block cert page %v", err) + return "Blocked cert default page" + } + return res } -func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) (*string, int) { +func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) (category *string, matchType int, isRelaxedPolicy bool) { if am.RulesCnt == 0 { - return nil, Included + return nil, Included, false } cacheKey := url + host @@ -109,25 +156,25 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string item := v.(cacheItem) log.Printf("Cache hit: %s %d", url, item.matchType) - return item.category, item.matchType + return item.category, item.matchType, item.isRelaxedPolicy } res1, res2 := am.matchRulesCategories(am.MatcherCategories, url, host, referer) if res1 != nil { - lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2}) - return res1, res2 + lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2, isRelaxedPolicy: false}) + return res1, res2, false } if am.bypassEnabled { - return nil, Included + return nil, Included, true } res1, res2 = am.matchRulesCategories(am.BypassMatcherCategories, url, host, referer) if res1 != nil { - lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2}) + lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2, isRelaxedPolicy: true}) } - lruCache.Add(cacheKey, cacheItem{category: nil, matchType: Included}) - return res1, res2 + lruCache.Add(cacheKey, cacheItem{category: nil, matchType: Included, isRelaxedPolicy: true}) + return res1, res2, true } func (am *AdBlockMatcher) matchRulesCategories(matcherCategories []*MatcherCategory, url string, host string, referer string) (*string, int) { diff --git a/goproxy/adblock_import.go b/goproxy/adblock_import.go index 76cc84d..ae929ad 100644 --- a/goproxy/adblock_import.go +++ b/goproxy/adblock_import.go @@ -7,6 +7,7 @@ import ( "log" "strings" + "github.com/aymerick/raymond" "github.com/patriciy/adblock/adblock" ) @@ -96,7 +97,8 @@ func (am *AdBlockMatcher) addBlockPageFromZipFile(file *zip.File) { if e != nil { log.Printf("Error reading block page %s %s", e, file.Name) } - am.BlockPageContent = string(content) + blockPageContent := string(content) + am.BlockPageTemplate, _ = raymond.Parse(blockPageContent) } func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, whitelist bool, bypass bool) { diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index f3137f0..18f7d5d 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -4,6 +4,7 @@ import "C" import ( "bufio" + "github.com/aymerick/raymond" "io/ioutil" "log" "os" @@ -64,20 +65,46 @@ func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listTyp } //export AdBlockMatcherSetBlockedPageContent -func AdBlockMatcherSetBlockedPageContent(contentC *C.char) { - blockPagePath := C.GoString(contentC) - fileHandle, err := os.Open(blockPagePath) +func AdBlockMatcherSetBlockedPageContent(contentBlockPageC, contentCertPageC *C.char) { + blockPagePath := C.GoString(contentBlockPageC) + adBlockMatcher.BlockPageTemplate = parseTemplate(blockPagePath) + + certPagePath := C.GoString(contentCertPageC) + adBlockMatcher.BlockCertTemplate = parseTemplate(certPagePath) +} + +//export AdBlockMatcherSetBlockPageContextTag +func AdBlockMatcherSetBlockPageContextTag(keyC, valueC *C.char) { + key := C.GoString(keyC) + value := C.GoString(valueC) + + if len(value) > 0 { + adBlockMatcher.defaultBlockPageTags[key] = value + } else { + delete(adBlockMatcher.defaultBlockPageTags, key) + } +} + +func parseTemplate(pagePath string) *raymond.Template { + fileHandle, err := os.Open(pagePath) if err != nil { log.Printf("Error reading block page %s", err) - return + return nil } defer fileHandle.Close() content, e := ioutil.ReadAll(fileHandle) if e != nil { log.Printf("Error reading block page %s", e) - return + return nil } - adBlockMatcher.BlockPageContent = string(content) + + pageString := string(content) + template, err := raymond.Parse(pageString) + if err != nil { + log.Printf("Error parsing template %s, %v", pagePath, err) + return nil + } + return template } //export AdBlockMatcherSave @@ -92,12 +119,16 @@ func AdBlockMatcherLoad(fileName string) { //export AdBlockMatcherEnableBypass func AdBlockMatcherEnableBypass() { - adBlockMatcher.bypassEnabled = true + if adBlockMatcher != nil { + adBlockMatcher.bypassEnabled = true + } } //export AdBlockMatcherDisableBypass func AdBlockMatcherDisableBypass() { - adBlockMatcher.bypassEnabled = false + if adBlockMatcher != nil { + adBlockMatcher.bypassEnabled = false + } } //export AdBlockMatcherGetBypassEnabled From 7f7764af40f29697c4fedb6451da75498011ac44 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 2 Apr 2020 16:57:04 +0700 Subject: [PATCH 05/27] missing files --- goproxy/goproxystarter.go | 239 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 goproxy/goproxystarter.go diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go new file mode 100644 index 0000000..78bbf90 --- /dev/null +++ b/goproxy/goproxystarter.go @@ -0,0 +1,239 @@ +package main + +/* +#include + +typedef int (*adBlockCallback)(char* url, char* category); + +static inline int FireAdblockCallback(void* ptr, char* url, char* category) +{ + adBlockCallback p = (adBlockCallback)ptr; + return p(url, category); +} +*/ +import "C" + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "time" + "unsafe" + + "github.com/elazarl/goproxy" + "github.com/inconshreveable/go-vhost" +) + +//import _ "net/http/pprof" + +var ( + proxy *goproxy.ProxyHttpServer + server *http.Server +) + +func initGoProxy() { + proxy = goproxy.NewProxyHttpServer() + proxy.Verbose = true + + proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host == "" { + fmt.Fprintln(w, "Cannot handle requests without Host header, e.g., HTTP 1.0") + return + } + + req.URL.Scheme = "http" + req.URL.Host = req.Host + proxy.ServeHTTP(w, req) + }) + + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + + if proxy.Verbose { + log.Printf("Server inited") + } +} + +type dumbResponseWriter struct { + net.Conn +} + +func (dumb dumbResponseWriter) Header() http.Header { + // panic("Header() should not be called on this ResponseWriter") + return make(http.Header) +} + +func (dumb dumbResponseWriter) Write(buf []byte) (int, error) { + if bytes.Equal(buf, []byte("HTTP/1.0 200 OK\r\n\r\n")) { + return len(buf), nil // throw away the HTTP OK response from the faux CONNECT request + } + return dumb.Conn.Write(buf) +} + +func (dumb dumbResponseWriter) WriteHeader(code int) { + // panic("WriteHeader() should not be called on this ResponseWriter") +} + +func (dumb dumbResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return dumb, bufio.NewReadWriter(bufio.NewReader(dumb), bufio.NewWriter(dumb)), nil +} + +func runHttpsListener(port int16) { + // listen to the TLS ClientHello but make it a CONNECT request instead + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + + if err != nil { + log.Printf("Error listening for https connections - %v", err) + return + } + + for { + c, err := ln.Accept() + if err != nil { + log.Printf("Error accepting new connection - %v", err) + continue + } + + go func(c net.Conn) { + tlsConn, err := vhost.TLS(c) + if err != nil { + log.Printf("Error accepting new connection - %v", err) + } + if tlsConn.Host() == "" { + log.Printf("Cannot support non-SNI enabled clients") + return + } + + if proxy.Verbose { + log.Printf("Https handler called for %s", tlsConn.Host()) + } + + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{ + Opaque: tlsConn.Host(), + Host: net.JoinHostPort(tlsConn.Host(), "443"), + }, + Host: tlsConn.Host(), + Header: make(http.Header), + } + + resp := dumbResponseWriter{tlsConn} + proxy.ServeHTTP(resp, connectReq) + }(c) + } +} + +func startHttpServer(port int16) *http.Server { + srv := &http.Server{Addr: fmt.Sprintf(":%d", port)} + srv.Handler = proxy + + // go func() { + // http.ListenAndServe(":6060", nil) + // }() + go func() { + if err := srv.ListenAndServe(); err != nil { + // cannot panic, because this probably is an intentional close + log.Printf("Httpserver: ListenAndServe() error: %s", err) + server = nil + } + }() + + // returning reference so caller can call Shutdown() + return srv +} + +func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) { + initGoProxy() + loadAndSetCa(certPath, certKeyPath) + + if proxy == nil { + return + } + + if proxy.Verbose { + log.Printf("Server is about to start http: %d, https: %d", portHttp, portHttps) + } + + server = startHttpServer(portHttp) + + proxy.OnRequest().DoFunc( + func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if adBlockMatcher != nil { + category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(r.URL.String(), r.Host, r.Referer()) + if category != nil && matchType == Included { + url := r.URL.String() + if adBlockBlacklistCallback != nil { + unsafeUrl := C.CString(url) + unsafeCategory := C.CString(*category) + C.FireAdblockCallback(adBlockBlacklistCallback, unsafeUrl, unsafeCategory) + C.free(unsafe.Pointer(unsafeUrl)) + C.free(unsafe.Pointer(unsafeCategory)) + } + + return r, goproxy.NewResponse(r, + goproxy.ContentTypeHtml, http.StatusForbidden, + adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy, false)) + } + } + return r, nil + }) + + proxy.OnResponse().DoFunc( + func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + if resp == nil { + return resp + } + + if resp.StatusCode > 400 { //ignore errors + return resp + } + + if adBlockMatcher == nil { + return resp + } + + if !adBlockMatcher.TestContentTypeIsFiltrable(resp.Header.Get("Content-Type")) { + return resp + } + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + + bytesData := buf.Bytes() + + //since we'd read all body - we need to recreate reader for client here + resp.Body.Close() + resp.Body = ioutil.NopCloser(bytes.NewBuffer(bytesData)) + + if !adBlockMatcher.IsContentSmallEnoughToFilter(int64(len(bytesData))) { + return resp + } + + category := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) + + if category != nil { + message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false, true) + return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + } + return resp + }) + + go runHttpsListener(portHttps) + + if proxy.Verbose { + log.Printf("Server started") + } +} + +func stopGoProxyServer() { + if server != nil { + context, _ := context.WithTimeout(context.Background(), 1*time.Second) + server.Shutdown(context) + server = nil + } +} From 7eebc35cd837f99f3bc7d968d43cd1fdf2711173 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 2 Apr 2020 20:39:40 +0700 Subject: [PATCH 06/27] http2 merged --- goproxy/certs.go | 2 +- goproxy/goproxystarter.go | 99 ++++++++++++++++++++++++++++++++------- goproxy/response.go | 2 +- 3 files changed, 83 insertions(+), 20 deletions(-) diff --git a/goproxy/certs.go b/goproxy/certs.go index 864e02e..747bd9e 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -16,7 +16,7 @@ import ( "os" "time" - "github.com/elazarl/goproxy" + "github.com/cloudveiltech/goproxy" ) var defaultTLSConfig = &tls.Config{ diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 78bbf90..4e954c2 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -14,29 +14,37 @@ static inline int FireAdblockCallback(void* ptr, char* url, char* category) import "C" import ( + "encoding/binary" "bufio" "bytes" "context" "fmt" + "io" "io/ioutil" "log" "net" "net/http" "net/url" + "strconv" "time" "unsafe" - "github.com/elazarl/goproxy" + "github.com/cloudveiltech/goproxy" "github.com/inconshreveable/go-vhost" ) //import _ "net/http/pprof" var ( - proxy *goproxy.ProxyHttpServer - server *http.Server + proxy *goproxy.ProxyHttpServer + server *http.Server + configuredPortHttp int16 + configuredPortHttps int16 ) +const DEFAULT_HTTPS_PORT uint16 = 443 + + func initGoProxy() { proxy = goproxy.NewProxyHttpServer() proxy.Verbose = true @@ -83,12 +91,14 @@ func (dumb dumbResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { return dumb, bufio.NewReadWriter(bufio.NewReader(dumb), bufio.NewWriter(dumb)), nil } -func runHttpsListener(port int16) { +func runHttpsListener() { + log.Printf("runHttpsListener() %d", configuredPortHttps) + // listen to the TLS ClientHello but make it a CONNECT request instead - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", configuredPortHttps)) if err != nil { - log.Printf("Error listening for https connections - %v", err) + log.Fatalf("Error listening for https connections - %v", err) return } @@ -100,38 +110,50 @@ func runHttpsListener(port int16) { } go func(c net.Conn) { + helloBuffer := make([]byte, 2) + n, err := c.Read(helloBuffer) + + port := DEFAULT_HTTPS_PORT + if n > 0 { + port = binary.BigEndian.Uint16([]byte{ helloBuffer[1], helloBuffer[0] }) + log.Printf("Reading dest port for %d", port) + + } + tlsConn, err := vhost.TLS(c) if err != nil { - log.Printf("Error accepting new connection - %v", err) - } - if tlsConn.Host() == "" { - log.Printf("Cannot support non-SNI enabled clients") + log.Printf("Assuming plain http connection - %v", err) + chainReqToHttp(tlsConn) return } - if proxy.Verbose { - log.Printf("Https handler called for %s", tlsConn.Host()) + host := tlsConn.Host() + if host == "" { + log.Printf("Cannot support client") + return } + host = net.JoinHostPort(host, strconv.Itoa(int(port))) + resp := dumbResponseWriter{tlsConn} connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{ - Opaque: tlsConn.Host(), - Host: net.JoinHostPort(tlsConn.Host(), "443"), + Opaque: host, + Host: host, }, - Host: tlsConn.Host(), + Host: host, Header: make(http.Header), } - resp := dumbResponseWriter{tlsConn} proxy.ServeHTTP(resp, connectReq) }(c) } } func startHttpServer(port int16) *http.Server { - srv := &http.Server{Addr: fmt.Sprintf(":%d", port)} + srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port)} srv.Handler = proxy + proxy.Http2Handler = serveHttp2Filtering // go func() { // http.ListenAndServe(":6060", nil) @@ -151,6 +173,8 @@ func startHttpServer(port int16) *http.Server { func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) { initGoProxy() loadAndSetCa(certPath, certKeyPath) + configuredPortHttp = portHttp + configuredPortHttps = portHttps if proxy == nil { return @@ -223,13 +247,52 @@ func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) return resp }) - go runHttpsListener(portHttps) + go runHttpsListener() if proxy.Verbose { log.Printf("Server started") } } +func chainReqToHttp(client net.Conn) { + remote, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", configuredPortHttp)) + if err != nil { + log.Printf("chainReqToHttp error connect %s", err) + return + } + + defer remote.Close() + defer client.Close() + + go func() { + for { + n, err := io.Copy(remote, client) + if err != nil { + log.Printf("error request %s", err) + return + } + if n == 0 { + log.Printf("nothing requested close") + return + } + time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop + } + }() + + for { + n, err := io.Copy(client, remote) + if err != nil { + log.Printf("error response %s", err) + return + } + if n == 0 { + log.Printf("nothing responded close") + return + } + time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop + } +} + func stopGoProxyServer() { if server != nil { context, _ := context.WithTimeout(context.Background(), 1*time.Second) diff --git a/goproxy/response.go b/goproxy/response.go index 7ccae77..58d10cc 100644 --- a/goproxy/response.go +++ b/goproxy/response.go @@ -8,7 +8,7 @@ import ( "io/ioutil" "strings" - "github.com/elazarl/goproxy" + "github.com/cloudveiltech/goproxy" ) //export ResponseGetStatusCode From 70b78d1b05445ce12b467d64c0ced0398ec608cf Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 17 Apr 2020 20:38:37 +0700 Subject: [PATCH 07/27] add conf https server, add cert verification --- goproxy/adblock.go | 9 +++++---- goproxy/certs.go | 7 ++++++- goproxy/goproxystarter.go | 26 +++++++++++++++++++++----- goproxy/main.go | 22 +++++++++++++++++++--- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index bd1e944..f7ed7e9 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -128,11 +128,12 @@ func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPol return res } -func (am *AdBlockMatcher) GetBadCertPage(url, category, host, certThumbPrint string) string { +func (am *AdBlockMatcher) GetBadCertPage(host, certThumbPrint string) string { tags := am.defaultBlockPageTags - tags["url_text"] = url - tags["friendly_url_text"] = url - tags["certThumbprintExists"] = certThumbPrint + if len(certThumbPrint) > 0 { + tags["certThumbprintExists"] = "1" + } + tags["certThumbprint"] = certThumbPrint tags["host"] = host if am.BlockCertTemplate == nil { diff --git a/goproxy/certs.go b/goproxy/certs.go index 747bd9e..df03dfd 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -14,6 +14,7 @@ import ( "log" "math/big" "os" + "strings" "time" "github.com/cloudveiltech/goproxy" @@ -85,10 +86,13 @@ func setCA(caCert, caKey []byte) error { } func verifyCerts(dnsName string, peerCerts []*x509.Certificate) (bool, error) { + dnsNamePatched := strings.Split(dnsName, ":")[0] + opts := x509.VerifyOptions{ Roots: nil, - DNSName: dnsName, + DNSName: dnsNamePatched, Intermediates: x509.NewCertPool(), + CurrentTime: time.Now(), } for i, cert := range peerCerts { @@ -102,6 +106,7 @@ func verifyCerts(dnsName string, peerCerts []*x509.Certificate) (bool, error) { var err error _, err = peerCerts[0].Verify(opts) if err != nil { + log.Printf("Verify certs error %v", err) return false, err } diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 4e954c2..af4750f 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -14,10 +14,10 @@ static inline int FireAdblockCallback(void* ptr, char* url, char* category) import "C" import ( - "encoding/binary" "bufio" "bytes" "context" + "encoding/binary" "fmt" "io" "io/ioutil" @@ -28,6 +28,7 @@ import ( "strconv" "time" "unsafe" + "encoding/base64" "github.com/cloudveiltech/goproxy" "github.com/inconshreveable/go-vhost" @@ -40,11 +41,11 @@ var ( server *http.Server configuredPortHttp int16 configuredPortHttps int16 + configuredConfigurationServerPort int16 ) const DEFAULT_HTTPS_PORT uint16 = 443 - func initGoProxy() { proxy = goproxy.NewProxyHttpServer() proxy.Verbose = true @@ -110,12 +111,12 @@ func runHttpsListener() { } go func(c net.Conn) { - helloBuffer := make([]byte, 2) + helloBuffer := make([]byte, 2) n, err := c.Read(helloBuffer) port := DEFAULT_HTTPS_PORT if n > 0 { - port = binary.BigEndian.Uint16([]byte{ helloBuffer[1], helloBuffer[0] }) + port = binary.BigEndian.Uint16([]byte{helloBuffer[1], helloBuffer[0]}) log.Printf("Reading dest port for %d", port) } @@ -170,11 +171,12 @@ func startHttpServer(port int16) *http.Server { return srv } -func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) { +func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, certPath, certKeyPath string) { initGoProxy() loadAndSetCa(certPath, certKeyPath) configuredPortHttp = portHttp configuredPortHttps = portHttps + configuredConfigurationServerPort = portConfigurationServer if proxy == nil { return @@ -222,6 +224,19 @@ func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) return resp } + if resp.TLS != nil { + var err error + _, err = verifyCerts(ctx.Req.URL.Host, resp.TLS.PeerCertificates) + if err != nil { + certThumbPrint := base64.StdEncoding.EncodeToString(resp.TLS.PeerCertificates[0].Signature) + + if !isCertInException(certThumbPrint) { + message := adBlockMatcher.GetBadCertPage(ctx.Req.URL.Host, certThumbPrint) + return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + } + } + } + if !adBlockMatcher.TestContentTypeIsFiltrable(resp.Header.Get("Content-Type")) { return resp } @@ -247,6 +262,7 @@ func startGoProxyServer(portHttp, portHttps int16, certPath, certKeyPath string) return resp }) + runConfigurationServerListener() go runHttpsListener() if proxy.Verbose { diff --git a/goproxy/main.go b/goproxy/main.go index fd64896..630861a 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -17,6 +17,22 @@ const ( ERROR_CERTS_GENERATION = -2 ) +var certsException = make(map[string]bool) + +//export AddCertException +func AddCertException(thumbPrintC *C.char) { + thumbPrint := C.GoString(thumbPrintC) + _, ok := certsException[thumbPrint] + if !ok { + certsException[thumbPrint] = true + } +} + +func isCertInException(thumbPrint string) bool { + _, ok := certsException[thumbPrint] + return ok +} + func checkPortAvailable(port int16) bool { l, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) defer l.Close() @@ -45,7 +61,7 @@ func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { } //export StartGoServer -func StartGoServer(portHttp int16, portHttps int16, certFileC *C.char, keyFileC *C.char) int16 { +func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char) int16 { debug.SetTraceback("all") debug.SetPanicOnFault(true) @@ -63,7 +79,7 @@ func StartGoServer(portHttp int16, portHttps int16, certFileC *C.char, keyFileC } } - startGoProxyServer(portHttp, portHttps, certFile, keyFile) + startGoProxyServer(portHttp, portHttps, portConfigurationServer, certFile, keyFile) return SUCCESS } @@ -78,7 +94,7 @@ func main() { func test() { log.Printf("main: starting HTTP server") - startGoProxyServer(14500, 14501, "rootCertificate.pem", "rootPrivateKey.pem") + startGoProxyServer(14500, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") log.Printf("main: serving for 1000 seconds") From 620413b6ea55e24a2b7778d0f5cd8c6a584175aa Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Mon, 20 Apr 2020 16:17:50 +0700 Subject: [PATCH 08/27] new interop api for whitelist matching --- goproxy/adblock.go | 11 +++++++++++ goproxy/adblock_interop.go | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index f7ed7e9..d1954ed 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -147,6 +147,17 @@ func (am *AdBlockMatcher) GetBadCertPage(host, certThumbPrint string) string { return res } +func (am *AdBlockMatcher) IsDomainWhitelisted(host string) bool { + category, matchType, _ := am.TestUrlBlocked("https://"+host, host, "") + if category != nil && matchType == Excluded { + log.Printf("Testing early host - true %s", host) + return true + } + + log.Printf("Testing early host - false %s", host) + return false +} + func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) (category *string, matchType int, isRelaxedPolicy bool) { if am.RulesCnt == 0 { return nil, Included, false diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 18f7d5d..c6ba035 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -139,3 +139,13 @@ func AdBlockMatcherGetBypassEnabled() bool { return false } } + +//export AdBlockMatcherIsDomainWhitelisted +func AdBlockMatcherIsDomainWhitelisted(hostC *C.char) bool { + if adBlockMatcher != nil { + host := C.GoString(hostC) + return adBlockMatcher.IsDomainWhitelisted(host) + } else { + return false + } +} From d35b065bb765708b0f9a243851e3d0bb2f36e212 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 3 Jul 2020 15:17:22 +0700 Subject: [PATCH 09/27] add url to bad cert bugfix --- goproxy/adblock.go | 40 ++++++++++------------------ goproxy/adblock_interop.go | 10 +++++-- goproxy/goproxystarter.go | 24 ++++++++++------- goproxy/response.go | 53 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index d1954ed..d6ddf6e 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -15,7 +15,6 @@ import ( goahocorasick "github.com/anknown/ahocorasick" "github.com/aymerick/raymond" - lru "github.com/hashicorp/golang-lru" ) const ( @@ -28,7 +27,6 @@ const MAX_CONTENT_SIZE_SCAN = 1000 * 1024 //500kb max to scan var adBlockMatcher *AdBlockMatcher var defaultBlockPageContent = "{{url_text}} is blocked. Category {{matching_category}}. Reason {{message}}" -var lruCache, _ = lru.New(1024) type cacheItem struct { category *string @@ -99,7 +97,7 @@ func (am *AdBlockMatcher) addMatcher(category string, bypass bool) { adBlockMatcher.lastCategory = categoryMatcher } -func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPolicy, isTriggerBlocked bool) string { +func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPolicy bool) string { tags := am.defaultBlockPageTags tags["url_text"] = blockedUrl @@ -111,12 +109,7 @@ func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPol } else { tags["isRelaxedPolicy"] = "" } - - if isTriggerBlocked { - tags["showUnblockRequestButton"] = "" - } else { - tags["showUnblockRequestButton"] = "1" - } + tags["showUnblockRequestButton"] = "1" tags["unblockRequest"] = tags["unblockRequestBase"] + "&category_name=" + url.QueryEscape(category) + "&blocked_request=" + base64.StdEncoding.EncodeToString([]byte(blockedUrl)) @@ -128,11 +121,14 @@ func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPol return res } -func (am *AdBlockMatcher) GetBadCertPage(host, certThumbPrint string) string { +func (am *AdBlockMatcher) GetBadCertPage(blockedUrl, host, certThumbPrint string) string { tags := am.defaultBlockPageTags if len(certThumbPrint) > 0 { tags["certThumbprintExists"] = "1" } + + tags["url_text"] = blockedUrl + tags["friendly_url_text"] = blockedUrl tags["certThumbprint"] = certThumbPrint tags["host"] = host @@ -163,16 +159,8 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string return nil, Included, false } - cacheKey := url + host - if v, ok := lruCache.Get(cacheKey); ok { - item := v.(cacheItem) - - log.Printf("Cache hit: %s %d", url, item.matchType) - return item.category, item.matchType, item.isRelaxedPolicy - } res1, res2 := am.matchRulesCategories(am.MatcherCategories, url, host, referer) if res1 != nil { - lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2, isRelaxedPolicy: false}) return res1, res2, false } @@ -181,11 +169,7 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string } res1, res2 = am.matchRulesCategories(am.BypassMatcherCategories, url, host, referer) - if res1 != nil { - lruCache.Add(cacheKey, cacheItem{category: res1, matchType: res2, isRelaxedPolicy: true}) - } - lruCache.Add(cacheKey, cacheItem{category: nil, matchType: Included, isRelaxedPolicy: true}) return res1, res2, true } @@ -221,7 +205,6 @@ func (am *AdBlockMatcher) matchRulesCategories(matcherCategories []*MatcherCateg func matchDomain(domainParts []string, matcherCatergory *MatcherCategory) (bool, int) { partsLen := len(domainParts) if partsLen < 2 { - log.Printf("Domain too short") return false, Included } domainName := domainParts[partsLen-1] @@ -249,21 +232,26 @@ func (am *AdBlockMatcher) IsContentSmallEnoughToFilter(contentSize int64) bool { return contentSize > 0 && contentSize < MAX_CONTENT_SIZE_SCAN } -func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) *string { +func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) (*string, []string) { text := []rune(strings.ToLower(string(str))) for _, phraseCategory := range am.PhraseCategories { if phraseCategory.processor == nil { + log.Printf("Searching text trigger: nil") continue } res := phraseCategory.processor.MultiPatternSearch(text, true) if len(res) > 0 { - return &phraseCategory.Category + words := make([]string, len(res)) + for i, term := range res { + words[i] = string(term.Word) + } + return &phraseCategory.Category, words } } - return nil + return nil, nil } func (am *AdBlockMatcher) AddBlockedPhrase(phrase string, category string) { diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index c6ba035..05327f0 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -4,11 +4,12 @@ import "C" import ( "bufio" - "github.com/aymerick/raymond" "io/ioutil" "log" "os" "unsafe" + + "github.com/aymerick/raymond" ) var adBlockBlacklistCallback unsafe.Pointer @@ -35,8 +36,13 @@ func AdBlockMatcherInitialize() { if oldMatcher != nil { adBlockMatcher.bypassEnabled = oldMatcher.bypassEnabled } +} - lruCache.Purge() +//export AdBlockMatcherBuild +func AdBlockMatcherBuild() { + if adBlockMatcher != nil { + adBlockMatcher.Build() + } } //export AdBlockMatcherParseRuleFile diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index af4750f..6829852 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -17,6 +17,7 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/binary" "fmt" "io" @@ -26,9 +27,9 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "unsafe" - "encoding/base64" "github.com/cloudveiltech/goproxy" "github.com/inconshreveable/go-vhost" @@ -37,10 +38,10 @@ import ( //import _ "net/http/pprof" var ( - proxy *goproxy.ProxyHttpServer - server *http.Server - configuredPortHttp int16 - configuredPortHttps int16 + proxy *goproxy.ProxyHttpServer + server *http.Server + configuredPortHttp int16 + configuredPortHttps int16 configuredConfigurationServerPort int16 ) @@ -202,9 +203,11 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert C.free(unsafe.Pointer(unsafeCategory)) } + log.Printf("Page %s blocked by url, category %s", url, *category) + return r, goproxy.NewResponse(r, goproxy.ContentTypeHtml, http.StatusForbidden, - adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy, false)) + adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy)) } } return r, nil @@ -231,7 +234,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert certThumbPrint := base64.StdEncoding.EncodeToString(resp.TLS.PeerCertificates[0].Signature) if !isCertInException(certThumbPrint) { - message := adBlockMatcher.GetBadCertPage(ctx.Req.URL.Host, certThumbPrint) + message := adBlockMatcher.GetBadCertPage(ctx.Req.URL.String(), ctx.Req.URL.Host, certThumbPrint) return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) } } @@ -253,10 +256,13 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert return resp } - category := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) + bytesData = decodeResponseCompression(resp.Header.Get("Content-Encoding"), bytesData) + + category, matches := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) if category != nil { - message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false, true) + log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", resp.Request.URL.String(), *category, strings.Join(matches, ", ")) + message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false) return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) } return resp diff --git a/goproxy/response.go b/goproxy/response.go index 58d10cc..3922272 100644 --- a/goproxy/response.go +++ b/goproxy/response.go @@ -5,10 +5,16 @@ import ( ) import ( "bytes" + "io" "io/ioutil" + "log" "strings" + "compress/flate" + "compress/gzip" + "github.com/cloudveiltech/goproxy" + "github.com/dsnet/compress/brotli" ) //export ResponseGetStatusCode @@ -35,11 +41,19 @@ func ResponseGetBody(id int64, res *[]byte) bool { buf := new(bytes.Buffer) buf.ReadFrom(response.Body) - *res = buf.Bytes() + result := buf.Bytes() + if response.Uncompressed { + *res = result + } else { + *res = decodeResponseCompression(response.Header.Get("Content-Encoding"), result) + if *res == nil { + *res = result + } + } //since we'd read all body - we need to recreate reader for client here response.Body.Close() - response.Body = ioutil.NopCloser(bytes.NewBuffer(*res)) + response.Body = ioutil.NopCloser(bytes.NewBuffer(result)) return true } @@ -55,6 +69,40 @@ func ResponseGetBodyAsString(id int64, res *string) bool { return true } +func decodeResponseCompression(contentEncoding string, body []byte) []byte { + switch contentEncoding { + case "gzip": + reader, err := gzip.NewReader(bytes.NewBuffer(body)) + return readReader(reader, err) + case "br": + reader, err := brotli.NewReader(bytes.NewBuffer(body), nil) + if err == nil { + buf := make([]byte, 1024) + body = make([]byte, 0) + defer reader.Close() + n, _ := reader.Read(buf) + for n > 0 { + body = append(body, buf...) + n, _ = reader.Read(buf) + } + return body + } + case "deflate": + reader := flate.NewReader(bytes.NewBuffer(body)) + return readReader(reader, nil) + } + return body +} + +func readReader(reader io.ReadCloser, err error) []byte { + if err == nil { + defer reader.Close() + body, _ := ioutil.ReadAll(reader) + return body + } + return nil +} + //export ResponseHasBody func ResponseHasBody(id int64) bool { response := getSessionResponse(id) @@ -158,6 +206,7 @@ func ResponseGetCertificate(id int64, index int32, certData *[]byte) int { func CreateResponse(id int64, status int32, contentType string, body string) bool { session, exists := sessionMap[id] if !exists { + log.Printf("NO SESSION FOR RESPONSE %d", id) return false } From 592dd9c96d7c8e6a5bd553cd2fc3d86a03f7bf77 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 9 Jul 2020 18:06:17 +0700 Subject: [PATCH 10/27] check whole-word match for triggers add log rotation --- goproxy/adblock.go | 33 ++++++++++++++++++++---- goproxy/goproxystarter.go | 1 + goproxy/main.go | 54 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index d6ddf6e..b9fd9c9 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -13,6 +13,8 @@ import ( "encoding/base64" + "unicode" + goahocorasick "github.com/anknown/ahocorasick" "github.com/aymerick/raymond" ) @@ -233,7 +235,8 @@ func (am *AdBlockMatcher) IsContentSmallEnoughToFilter(contentSize int64) bool { } func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) (*string, []string) { - text := []rune(strings.ToLower(string(str))) + originalText := strings.ToLower(string(str)) + text := []rune(originalText) for _, phraseCategory := range am.PhraseCategories { if phraseCategory.processor == nil { @@ -243,17 +246,37 @@ func (am *AdBlockMatcher) TestContainsForbiddenPhrases(str []byte) (*string, []s res := phraseCategory.processor.MultiPatternSearch(text, true) if len(res) > 0 { - words := make([]string, len(res)) - for i, term := range res { - words[i] = string(term.Word) + words := make([]string, 0) + for _, term := range res { + startIndex := term.Pos + endIndex := term.Pos + len(string(term.Word)) + + //check if there's whole word match + wholewordMatched := false + if startIndex > 0 && isNonLetterAndDigitRune(text[startIndex-1]) { + if endIndex < len(text)-2 && isNonLetterAndDigitRune(text[endIndex+1]) { + wholewordMatched = true + } + } + if wholewordMatched { + words = append(words, string(term.Word)) + } + } + if len(words) > 0 { + return &phraseCategory.Category, words + } else { + return nil, nil } - return &phraseCategory.Category, words } } return nil, nil } +func isNonLetterAndDigitRune(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) +} + func (am *AdBlockMatcher) AddBlockedPhrase(phrase string, category string) { var phraseCategory *PhraseCategory = nil for _, element := range adBlockMatcher.PhraseCategories { diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 6829852..f99ecea 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -191,6 +191,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert proxy.OnRequest().DoFunc( func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + monitorLogFileSize() if adBlockMatcher != nil { category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(r.URL.String(), r.Host, r.Referer()) if category != nil && matchType == Included { diff --git a/goproxy/main.go b/goproxy/main.go index 630861a..a615312 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -4,6 +4,7 @@ import "C" import ( "fmt" + "io" "log" "net" "os" @@ -15,14 +16,17 @@ const ( SUCCESS = 1 ERROR_PORTS_BUSY = -1 ERROR_CERTS_GENERATION = -2 + MAX_LOG_SIZE = 10 * 1024 * 1024 ) var certsException = make(map[string]bool) +var logFilePath = "" +var logFileHandle *os.File //export AddCertException func AddCertException(thumbPrintC *C.char) { thumbPrint := C.GoString(thumbPrintC) - _, ok := certsException[thumbPrint] + _, ok := certsException[thumbPrint] if !ok { certsException[thumbPrint] = true } @@ -46,13 +50,54 @@ func checkPortAvailable(port int16) bool { //export SetProxyLogFile func SetProxyLogFile(logFile *C.char) { - logPath := C.GoString(logFile) - file, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + logFilePath = C.GoString(logFile) + setProxyLogFileInternal(logFilePath) +} + +func setProxyLogFileInternal(logFile string) { + logFilePath = logFile + logFileHandle, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return + } + + redirectStderr(logFileHandle) +} + +func monitorLogFileSize() { + stat, err := os.Stat(logFilePath) if err != nil { + log.Printf("Can't stat log file %v", err) return } - redirectStderr(file) + if stat.Size() > MAX_LOG_SIZE { + log.Printf("Rotate log file") + destination, err := os.OpenFile(logFilePath+".1", os.O_CREATE|os.O_RDWR, 0777) + if err != nil { + log.Printf("Can't open temp file %v", err) + return + } + fileHandle, err := os.OpenFile(logFilePath, os.O_RDONLY, 0666) + if err != nil { + log.Printf("Can't open log file %v", err) + return + } + fileHandle.Seek(MAX_LOG_SIZE/2, 0) + + defer destination.Close() + _, err = io.Copy(destination, fileHandle) + if err != nil { + log.Printf("Can't copy log file %v", err) + return + } + + logFileHandle.Close() + os.Rename(logFilePath+".1", logFilePath) + setProxyLogFileInternal(logFilePath) + + log.Printf("Rotate log file done.") + } } //export AdBlockMatcherSetBlacklistCallback @@ -80,6 +125,7 @@ func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC } startGoProxyServer(portHttp, portHttps, portConfigurationServer, certFile, keyFile) + monitorLogFileSize() return SUCCESS } From 0add33f70bba2943ff609874facaab06c8dd38a2 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 10 Jul 2020 13:54:44 +0700 Subject: [PATCH 11/27] wihtelist rules go first --- goproxy/adblock.go | 54 ++++++++++++++++++++++++-------------- goproxy/adblock_import.go | 18 ++++++------- goproxy/adblock_interop.go | 4 +-- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index b9fd9c9..193ddfb 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -49,17 +49,18 @@ type PhraseCategory struct { } type AdBlockMatcher struct { - MatcherCategories []*MatcherCategory - BypassMatcherCategories []*MatcherCategory - PhraseCategories []*PhraseCategory - lastMatcher *adblock.RuleMatcher - lastCategory *MatcherCategory - RulesCnt int - phrasesCount int - bypassEnabled bool - BlockPageTemplate *raymond.Template - BlockCertTemplate *raymond.Template - defaultBlockPageTags map[string]string + WhiteListMatcherCategories []*MatcherCategory + BlackListMatcherCategories []*MatcherCategory + BypassMatcherCategories []*MatcherCategory + PhraseCategories []*PhraseCategory + lastMatcher *adblock.RuleMatcher + lastCategory *MatcherCategory + RulesCnt int + phrasesCount int + bypassEnabled bool + BlockPageTemplate *raymond.Template + BlockCertTemplate *raymond.Template + defaultBlockPageTags map[string]string } func CreateMatcher() *AdBlockMatcher { @@ -71,10 +72,17 @@ func CreateMatcher() *AdBlockMatcher { return adBlockMatcher } -func (am *AdBlockMatcher) addMatcher(category string, bypass bool) { +func (am *AdBlockMatcher) addMatcher(category string, listType int) { matcher := adblock.NewMatcher() var categoryMatcher *MatcherCategory - for _, element := range adBlockMatcher.MatcherCategories { + matcherCategories := adBlockMatcher.BlackListMatcherCategories + if listType == BypassList { + matcherCategories = adBlockMatcher.BypassMatcherCategories + } else if listType == Whitelist { + matcherCategories = adBlockMatcher.WhiteListMatcherCategories + } + + for _, element := range matcherCategories { if element.Category == category { categoryMatcher = element break @@ -87,10 +95,13 @@ func (am *AdBlockMatcher) addMatcher(category string, bypass bool) { BlockedDomains: make(map[string]bool), } - if bypass { + switch listType { + case Whitelist: + am.WhiteListMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) + case Blacklist: + am.BlackListMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) + case BypassList: am.BypassMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) - } else { - am.MatcherCategories = append(am.MatcherCategories, categoryMatcher) } } @@ -161,7 +172,12 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string return nil, Included, false } - res1, res2 := am.matchRulesCategories(am.MatcherCategories, url, host, referer) + res1, res2 := am.matchRulesCategories(am.WhiteListMatcherCategories, url, host, referer) + if res1 != nil { + return res1, res2, false + } + + res1, res2 = am.matchRulesCategories(am.BlackListMatcherCategories, url, host, referer) if res1 != nil { return res1, res2, false } @@ -312,10 +328,10 @@ func (am *AdBlockMatcher) Build() { am.phrasesCount += len(phraseCategory.Phrases) } - if len(am.MatcherCategories) == 0 { + if len(am.BlackListMatcherCategories) == 0 && len(am.WhiteListMatcherCategories) == 0 { return } - matchers := am.MatcherCategories[len(am.MatcherCategories)-1].Matchers + matchers := am.BlackListMatcherCategories[len(am.BlackListMatcherCategories)-1].Matchers am.lastMatcher = matchers[len(matchers)-1] debug.FreeOSMemory() diff --git a/goproxy/adblock_import.go b/goproxy/adblock_import.go index ae929ad..fbfdf75 100644 --- a/goproxy/adblock_import.go +++ b/goproxy/adblock_import.go @@ -23,7 +23,7 @@ func (am *AdBlockMatcher) ParseRulesZipArchive(filePath string) { } } -func (am *AdBlockMatcher) AddRule(rule string, category string, bypass bool) { +func (am *AdBlockMatcher) AddRule(rule string, category string, listType int) { r, e := adblock.ParseRule(rule) if e != nil { @@ -36,7 +36,7 @@ func (am *AdBlockMatcher) AddRule(rule string, category string, bypass bool) { } if am.RulesCnt%MAX_RULES_PER_MATCHER == 0 { - am.addMatcher(category, bypass) + am.addMatcher(category, listType) } //Check if it's just a domain rule @@ -73,13 +73,13 @@ func (am *AdBlockMatcher) ParseZipRulesFile(file *zip.File) { log.Printf("Opening triggers %s", file.Name) am.addPhrasesFromScanner(scanner, categoryName) } else if strings.Contains(file.Name, ".bypass") { - am.addMatcher(categoryName, true) + am.addMatcher(categoryName, BypassList) log.Printf("Opening bypass %s", file.Name) - am.addRulesFromScanner(scanner, categoryName, false, true) + am.addRulesFromScanner(scanner, categoryName, BypassList) } else if strings.Contains(file.Name, ".rules") { - am.addMatcher(categoryName, false) + am.addMatcher(categoryName, Blacklist) log.Printf("Opening rules %s", file.Name) - am.addRulesFromScanner(scanner, categoryName, false, false) + am.addRulesFromScanner(scanner, categoryName, Blacklist) } else { log.Printf("File type recognition failed %s", file.Name) } @@ -101,13 +101,13 @@ func (am *AdBlockMatcher) addBlockPageFromZipFile(file *zip.File) { am.BlockPageTemplate, _ = raymond.Parse(blockPageContent) } -func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, whitelist bool, bypass bool) { +func (am *AdBlockMatcher) addRulesFromScanner(scanner *bufio.Scanner, categoryName string, listType int) { for scanner.Scan() { line := scanner.Text() - if whitelist && !strings.HasPrefix(line, "@@") { + if listType == Whitelist && !strings.HasPrefix(line, "@@") { line = "@@" + line } - am.AddRule(line, categoryName, bypass) + am.AddRule(line, categoryName, listType) } } diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 05327f0..53859b4 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -60,12 +60,12 @@ func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listTyp log.Printf("Parsing category %s file %s", categoryId, fileName) - adBlockMatcher.addMatcher(categoryId, listType == BypassList) + adBlockMatcher.addMatcher(categoryId, int(listType)) if listType == TextTrigger { adBlockMatcher.addPhrasesFromScanner(scanner, categoryId) } else { - adBlockMatcher.addRulesFromScanner(scanner, categoryId, listType == Whitelist, listType == BypassList) + adBlockMatcher.addRulesFromScanner(scanner, categoryId, int(listType)) } return true } From ff18b16d3aa148a17eced4b63cce2b495fcca2eb Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 25 Sep 2020 18:12:48 +0700 Subject: [PATCH 12/27] added handleconnect callback --- goproxy/adblock.go | 4 +- goproxy/certs.go | 2 +- goproxy/goproxystarter.go | 87 +++++++++++++++++++++------------------ goproxy/main.go | 2 +- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 193ddfb..41c5841 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -97,9 +97,9 @@ func (am *AdBlockMatcher) addMatcher(category string, listType int) { switch listType { case Whitelist: - am.WhiteListMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) + am.WhiteListMatcherCategories = append(am.WhiteListMatcherCategories, categoryMatcher) case Blacklist: - am.BlackListMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) + am.BlackListMatcherCategories = append(am.BlackListMatcherCategories, categoryMatcher) case BypassList: am.BypassMatcherCategories = append(am.BypassMatcherCategories, categoryMatcher) } diff --git a/goproxy/certs.go b/goproxy/certs.go index df03dfd..c957ab4 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -157,7 +157,7 @@ func GenerateCerts(caCertPath, caKeyPath string) bool { NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour * 24 * 3650), - KeyUsage: x509.KeyUsageDigitalSignature, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, IsCA: true, diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index f99ecea..cc22933 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -47,6 +47,29 @@ var ( const DEFAULT_HTTPS_PORT uint16 = 443 +type HttpsHandler func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) + +func (f HttpsHandler) HandleConnect(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + return f(host, ctx) +} + +var handleConnectFunc HttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + hostWithoutPort := host + parts := strings.Split(hostWithoutPort, ":") + if len(parts) > 1 { + hostWithoutPort = strings.ReplaceAll(hostWithoutPort, parts[len(parts)-1], "") + log.Printf("Stripped port %s", hostWithoutPort) + } + + if adBlockMatcher.IsDomainWhitelisted(hostWithoutPort) { + log.Printf("Whitelisting host %s", host) + return goproxy.OkConnect, host + } + + log.Printf("Analyzing host %s", host) + return goproxy.MitmConnect, host +} + func initGoProxy() { proxy = goproxy.NewProxyHttpServer() proxy.Verbose = true @@ -62,7 +85,7 @@ func initGoProxy() { proxy.ServeHTTP(w, req) }) - proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest().HandleConnect(handleConnectFunc) if proxy.Verbose { log.Printf("Server inited") @@ -119,7 +142,6 @@ func runHttpsListener() { if n > 0 { port = binary.BigEndian.Uint16([]byte{helloBuffer[1], helloBuffer[0]}) log.Printf("Reading dest port for %d", port) - } tlsConn, err := vhost.TLS(c) @@ -152,6 +174,28 @@ func runHttpsListener() { } } +func chainReqToHttp(client net.Conn) { + chainReqToHost(client, fmt.Sprintf("127.0.0.1:%d", configuredPortHttp)) +} + +func chainReqToHost(client net.Conn, hostPort string) { + remote, err := net.Dial("tcp", hostPort) + + if err != nil { + log.Printf("chainReqToHttp error connect %s", err) + return + } + + // defer remote.Close() + // defer client.Close() + + go func() { + io.Copy(remote, client) + }() + + io.Copy(client, remote) +} + func startHttpServer(port int16) *http.Server { srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port)} srv.Handler = proxy @@ -277,45 +321,6 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert } } -func chainReqToHttp(client net.Conn) { - remote, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", configuredPortHttp)) - if err != nil { - log.Printf("chainReqToHttp error connect %s", err) - return - } - - defer remote.Close() - defer client.Close() - - go func() { - for { - n, err := io.Copy(remote, client) - if err != nil { - log.Printf("error request %s", err) - return - } - if n == 0 { - log.Printf("nothing requested close") - return - } - time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop - } - }() - - for { - n, err := io.Copy(client, remote) - if err != nil { - log.Printf("error response %s", err) - return - } - if n == 0 { - log.Printf("nothing responded close") - return - } - time.Sleep(time.Millisecond) //reduce CPU usage due to infinite nonblocking loop - } -} - func stopGoProxyServer() { if server != nil { context, _ := context.WithTimeout(context.Background(), 1*time.Second) diff --git a/goproxy/main.go b/goproxy/main.go index a615312..4c61a4e 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -140,7 +140,7 @@ func main() { func test() { log.Printf("main: starting HTTP server") - startGoProxyServer(14500, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") + startGoProxyServer(14600, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") log.Printf("main: serving for 1000 seconds") From 5d95bf33ad1ba8f1420f2ad5cee44dc0845e848b Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Sat, 26 Sep 2020 13:52:11 +0700 Subject: [PATCH 13/27] new interop methods --- goproxy/adblock.go | 20 ++++++++++++++++++++ goproxy/adblock_interop.go | 12 ++++++++++++ goproxy/goproxystarter.go | 4 +--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 41c5841..5d9bbd0 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -190,7 +190,27 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string return res1, res2, true } +func (am *AdBlockMatcher) GetWhitelistedDomains() []string { + res := make([]string, 0) + for _, matcherCategory := range am.WhiteListMatcherCategories { + for k, v := range matcherCategory.BlockedDomains { + if !v { + res = append(res, k) + } + } + } + + for _, matcherCategory := range am.BlackListMatcherCategories { + for k, v := range matcherCategory.BlockedDomains { + if !v { + res = append(res, k) + } + } + } + + return res +} func (am *AdBlockMatcher) matchRulesCategories(matcherCategories []*MatcherCategory, url string, host string, referer string) (*string, int) { rq := &adblock.Request{ URL: url, diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 53859b4..778b0a0 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "log" "os" + "strings" "unsafe" "github.com/aymerick/raymond" @@ -155,3 +156,14 @@ func AdBlockMatcherIsDomainWhitelisted(hostC *C.char) bool { return false } } + +//export AdBlockMatcherGetWhitelistedDomains +func AdBlockMatcherGetWhitelistedDomains() *C.char { + if adBlockMatcher != nil { + domains := adBlockMatcher.GetWhitelistedDomains() + res := strings.Join(domains, ";") + return C.CString(res) + } else { + return C.CString("") + } +} diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index cc22933..be7ab2e 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -57,8 +57,7 @@ var handleConnectFunc HttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (* hostWithoutPort := host parts := strings.Split(hostWithoutPort, ":") if len(parts) > 1 { - hostWithoutPort = strings.ReplaceAll(hostWithoutPort, parts[len(parts)-1], "") - log.Printf("Stripped port %s", hostWithoutPort) + hostWithoutPort = strings.ReplaceAll(hostWithoutPort, ":"+parts[len(parts)-1], "") } if adBlockMatcher.IsDomainWhitelisted(hostWithoutPort) { @@ -66,7 +65,6 @@ var handleConnectFunc HttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (* return goproxy.OkConnect, host } - log.Printf("Analyzing host %s", host) return goproxy.MitmConnect, host } From 75b3fceebf02594759d29a219d0b497dc192d049 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Mon, 10 May 2021 17:17:32 +0200 Subject: [PATCH 14/27] merge cv4w changes --- goproxy/adblock.go | 7 + goproxy/adblock_interop.go | 7 - goproxy/certs.go | 5 +- goproxy/goproxy_chain.go | 160 +++++++++++++ goproxy/goproxystarter.go | 9 +- goproxy/http2.go | 466 +++++++++++++++++++++++++++++++++++++ goproxy/main.go | 16 ++ goproxy/request.go | 15 ++ 8 files changed, 667 insertions(+), 18 deletions(-) create mode 100644 goproxy/goproxy_chain.go create mode 100644 goproxy/http2.go diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 5d9bbd0..664fddb 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -19,6 +19,13 @@ import ( "github.com/aymerick/raymond" ) +const ( + Blacklist = 1 + Whitelist = 2 + BypassList = 3 + TextTrigger = 4 +) + const ( Included = adblock.Included Excluded = adblock.Excluded diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 778b0a0..5a5e2ff 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -17,13 +17,6 @@ var adBlockBlacklistCallback unsafe.Pointer var adBlockMatchers map[int32]*AdBlockMatcher -const ( - Blacklist = 1 - Whitelist = 2 - BypassList = 3 - TextTrigger = 4 -) - //export AdBlockMatcherInitialize func AdBlockMatcherInitialize() { var oldMatcher *AdBlockMatcher = nil diff --git a/goproxy/certs.go b/goproxy/certs.go index c957ab4..b8a8e43 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -141,8 +141,7 @@ func pemBlockForKey(priv interface{}) *pem.Block { } func GenerateCerts(caCertPath, caKeyPath string) bool { - - // priv, err := rsa.GenerateKey(rand.Reader, *rsaBits) + //priv, err := rsa.GenerateKey(rand.Reader, 128) priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { log.Printf("Error generating cert %s", err) @@ -155,7 +154,7 @@ func GenerateCerts(caCertPath, caKeyPath string) bool { Organization: []string{"Cloudveil Filtering Certificate"}, }, NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 3650), + NotAfter: time.Now().Add(time.Hour * 24 * 365), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, diff --git a/goproxy/goproxy_chain.go b/goproxy/goproxy_chain.go new file mode 100644 index 0000000..4471e0e --- /dev/null +++ b/goproxy/goproxy_chain.go @@ -0,0 +1,160 @@ +package main + +import ( + "bytes" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "log" + "math/big" + "math/rand" + "net" + "net/http" + "net/url" + "runtime" + "sort" + "strings" + "time" + + "github.com/cloudveiltech/goproxy" +) + +func runConfigurationServerListener() { + go func() { + + cert, _ := signHost(goproxy.GoproxyCa, []string{"127.0.0.1"}) + config := defaultTLSConfig + config.Certificates = append(config.Certificates, *cert) + config.NextProtos = []string{"http/1.1"} + + srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", configuredConfigurationServerPort)} + srv.Handler = serverHandler{} + srv.TLSConfig = config + + srv.ListenAndServeTLS("", "") + }() +} + +type serverHandler struct { +} + +func (sh serverHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + localPort := fmt.Sprintf("%d", configuredConfigurationServerPort) + remotePort := fmt.Sprintf("%d", configuredConfigurationServerPort+1) + req.RequestURI = "http://127.0.0.1:" + remotePort + strings.ReplaceAll(req.RequestURI, localPort, remotePort) + req.URL, _ = url.ParseRequestURI(req.RequestURI) + req.Host = req.URL.Host + + log.Printf("Config server URI %s", req.RequestURI) + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + log.Printf("Config server err %v", err) + return + } + + for k, vv := range resp.Header { + if k != "Content-Length" { + for _, v := range vv { + log.Printf("Config server Response header %s:%s", k, v) + rw.Header().Add(k, v) + } + } + } + rw.WriteHeader(resp.StatusCode) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + rw.Write(buf.Bytes()) + log.Printf("Config server Response sent %s", req.RequestURI) +} + +func hashSorted(lst []string) []byte { + c := make([]string, len(lst)) + copy(c, lst) + sort.Strings(c) + h := sha1.New() + for _, s := range c { + h.Write([]byte(s + ",")) + } + return h.Sum(nil) +} + +func hashSortedBigInt(lst []string) *big.Int { + rv := new(big.Int) + rv.SetBytes(hashSorted(lst)) + return rv +} + +var goproxySignerVersion = ":goroxy1" + +func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err error) { + var x509ca *x509.Certificate + + // Use the provided ca and not the global GoproxyCa for certificate generation. + if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { + return + } + start := time.Unix(0, 0) + end, err := time.Parse("2006-01-02", "2049-12-31") + if err != nil { + panic(err) + } + + serial := big.NewInt(rand.Int63()) + template := x509.Certificate{ + // TODO(elazar): instead of this ugly hack, just encode the certificate and hash the binary form. + SerialNumber: serial, + Issuer: x509ca.Subject, + Subject: pkix.Name{ + Organization: []string{"GoProxy untrusted MITM proxy Inc"}, + }, + NotBefore: start, + NotAfter: end, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + template.Subject.CommonName = h + } + } + + hash := hashSorted(append(hosts, goproxySignerVersion, ":"+runtime.Version())) + var csprng goproxy.CounterEncryptorRand + if csprng, err = goproxy.NewCounterEncryptorRandFromKey(ca.PrivateKey, hash); err != nil { + return + } + + var certpriv crypto.Signer + switch ca.PrivateKey.(type) { + case *rsa.PrivateKey: + if certpriv, err = rsa.GenerateKey(&csprng, 2048); err != nil { + return + } + case *ecdsa.PrivateKey: + if certpriv, err = ecdsa.GenerateKey(elliptic.P256(), &csprng); err != nil { + return + } + default: + err = fmt.Errorf("unsupported key type %T", ca.PrivateKey) + } + + var derBytes []byte + if derBytes, err = x509.CreateCertificate(&csprng, &template, x509ca, certpriv.Public(), ca.PrivateKey); err != nil { + return + } + return &tls.Certificate{ + Certificate: [][]byte{derBytes, ca.Certificate[0]}, + PrivateKey: certpriv, + }, nil +} diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index be7ab2e..39aebe3 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -37,14 +37,6 @@ import ( //import _ "net/http/pprof" -var ( - proxy *goproxy.ProxyHttpServer - server *http.Server - configuredPortHttp int16 - configuredPortHttps int16 - configuredConfigurationServerPort int16 -) - const DEFAULT_HTTPS_PORT uint16 = 443 type HttpsHandler func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) @@ -248,6 +240,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert log.Printf("Page %s blocked by url, category %s", url, *category) + r.URL.RawPath = HostPathForceSafeSearch(r.URL.Host, r.URL.RawPath) return r, goproxy.NewResponse(r, goproxy.ContentTypeHtml, http.StatusForbidden, adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy)) diff --git a/goproxy/http2.go b/goproxy/http2.go new file mode 100644 index 0000000..373ddd9 --- /dev/null +++ b/goproxy/http2.go @@ -0,0 +1,466 @@ +package main + +import ( + "bytes" + "crypto/tls" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "sync/atomic" + + "github.com/cloudveiltech/goproxy" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +var http2ProxySessionCounter int64 + +const MIN_FILTERABLE_LENGTH = 100 + +type Http2Handler struct { + lastHttpResponse map[uint32]*http.Response + lastHttpRequest map[uint32]*http.Request + lastHeadersBlock map[uint32]*http2.HeadersFrameParam + proxyCtx map[uint32]*goproxy.ProxyCtx + lastHeadersMap map[uint32][]hpack.HeaderField + responseBodyMapChunks map[uint32][][]byte +} + +func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Conn) bool { + log.Print("Running http2 handler for " + r.URL.String()) + + http2Handler := &Http2Handler{ + lastHttpResponse: make(map[uint32]*http.Response), + lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), + lastHeadersMap: make(map[uint32][]hpack.HeaderField), + lastHttpRequest: make(map[uint32]*http.Request), + proxyCtx: make(map[uint32]*goproxy.ProxyCtx), + responseBodyMapChunks: make(map[uint32][][]byte), + } + go func() { + http2Handler.processHttp2Stream(rawClientTls, remote) + }() + + return true +} + +func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tls.Conn) { + const preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + b := make([]byte, len(preface)) + if _, err := io.ReadFull(local, b); err != nil { + log.Printf("ReadFrame: preface %v", err) + return + } + if string(b) != preface { + log.Printf("ReadFrame: preface error") + return + } + remote.Write(b) + + http2.VerboseLogs = false + directFramer := http2.NewFramer(remote, local) + reverseFramer := http2.NewFramer(local, remote) + + go func() { + defer remote.Close() + defer local.Close() + decoder := hpack.NewDecoder(65536, nil) + for { + if !http2Handler.readFrame(reverseFramer, directFramer, decoder, false) { + return + } + } + }() + + decoder := hpack.NewDecoder(65536, nil) + for { + if !http2Handler.readFrame(directFramer, reverseFramer, decoder, true) { + return + } + } +} + +func isContentTypeFilterable(contentType string) bool { + return strings.Contains(contentType, "html") || + strings.Contains(contentType, "json") +} + +func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { + f, err := directFramer.ReadFrame() + if err != nil { + log.Printf("ReadFrame client %v, err: %v", client, err) + return false + } + + switch f.Header().Type { + case http2.FrameData: + fr := f.(*http2.DataFrame) + body := fr.Data() + + lastHttpResponse := http2Handler.lastHttpResponse[f.Header().StreamID] + bodyChunks := http2Handler.responseBodyMapChunks[f.Header().StreamID] + chunk := make([]byte, len(body)) + copy(chunk, body) + bodyChunks = append(bodyChunks, chunk) + http2Handler.responseBodyMapChunks[f.Header().StreamID] = bodyChunks + + if lastHttpResponse != nil && !client { + contentType := lastHttpResponse.Header.Get("Content-Type") + isContentTypeFilterable := isContentTypeFilterable(contentType) + if isContentTypeFilterable && fr.StreamEnded() { + putResponseBody(bodyChunks, lastHttpResponse) + contentLength := lastHttpResponse.ContentLength + + if contentLength > MIN_FILTERABLE_LENGTH { + ctx := http2Handler.proxyCtx[f.Header().StreamID] + resp := proxy.FilterResponse(lastHttpResponse, ctx) + + if resp != lastHttpResponse { //new response + writeHeaders(directFramer, &http2.HeadersFrameParam{ + StreamID: f.Header().StreamID, + BlockFragment: encodeHeaders(resp), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: http2.PriorityParam{}, + }, decoder) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + directFramer.WriteData(f.Header().StreamID, true, buf.Bytes()) + directFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeCancel, nil) + delete(http2Handler.lastHttpResponse, f.Header().StreamID) + delete(http2Handler.lastHttpRequest, f.Header().StreamID) + delete(http2Handler.responseBodyMapChunks, f.Header().StreamID) + return false + } + } + } else if isContentTypeFilterable { + return true + } + } + + header, ok := http2Handler.lastHeadersBlock[f.Header().StreamID] + if ok { + // headerFields, _ := http2Handler.lastHeadersMap[f.Header().StreamID] + header.EndStream = false + // header.BlockFragment = encodeHeaderFields(headerFields) + writeHeaders(directFramer, header, decoder) + delete(http2Handler.lastHeadersBlock, f.Header().StreamID) + delete(http2Handler.lastHeadersMap, f.Header().StreamID) + } + + for i, _ := range bodyChunks { + streamEnded := i == len(bodyChunks)-1 && fr.StreamEnded() + directFramer.WriteData(f.Header().StreamID, streamEnded, bodyChunks[i]) + } + + delete(http2Handler.responseBodyMapChunks, f.Header().StreamID) + case http2.FrameHeaders: + fr := f.(*http2.HeadersFrame) + + headerFields, _ := decodeAllHeaders(directFramer, fr, decoder) + if len(headerFields) == 0 { + log.Printf("Error parsing headers") + } + writeHeadersImmediately := client || fr.StreamEnded() + if client { + request := makeHttpRequest(nil, headerFields) + var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1)} + http2Handler.lastHttpRequest[f.Header().StreamID] = request + http2Handler.proxyCtx[f.Header().StreamID] = ctx + _, resp := proxy.FilterRequest(request, ctx) + if resp != nil { + writeHeaders(reverseFramer, &http2.HeadersFrameParam{ + StreamID: f.Header().StreamID, + BlockFragment: encodeHeaders(resp), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: fr.Priority, + }, decoder) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + reverseFramer.WriteData(f.Header().StreamID, true, buf.Bytes()) + reverseFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeCancel, nil) + return false + } + } else { + response := makeHttpResponse(nil, headerFields) + http2Handler.lastHttpResponse[f.Header().StreamID] = response + contentType := response.Header.Get("Content-Type") + // contentLength, _ := strconv.Atoi(response.Header.Get("Content-Length")) + if !isContentTypeFilterable(contentType) { + writeHeadersImmediately = true + } + http2Handler.lastHttpResponse[f.Header().StreamID].Request = http2Handler.lastHttpRequest[f.Header().StreamID] + } + + header := http2.HeadersFrameParam{ + StreamID: f.Header().StreamID, + BlockFragment: encodeHeaderFields(headerFields), + EndStream: fr.StreamEnded(), + EndHeaders: fr.HeadersEnded(), + PadLength: 0, + Priority: fr.Priority, + } + + if writeHeadersImmediately { + writeHeaders(directFramer, &header, decoder) + } else { + http2Handler.lastHeadersMap[f.Header().StreamID] = headerFields + http2Handler.lastHeadersBlock[f.Header().StreamID] = &header + } + case http2.FramePriority: + fr := f.(*http2.PriorityFrame) + directFramer.WritePriority(f.Header().StreamID, fr.PriorityParam) + case http2.FrameRSTStream: + fr := f.(*http2.RSTStreamFrame) + directFramer.WriteRSTStream(f.Header().StreamID, fr.ErrCode) + case http2.FrameSettings: + fr := f.(*http2.SettingsFrame) + if fr.IsAck() { + directFramer.WriteSettingsAck() + } else { + params := make([]http2.Setting, 0) + for i := 0; i < fr.NumSettings(); i++ { + setting := fr.Setting(i) + params = append(params, setting) + if setting.ID == http2.SettingHeaderTableSize { + decoder.SetMaxDynamicTableSize(setting.Val) + } + } + directFramer.WriteSettings(params...) + } + case http2.FramePushPromise: + fr := f.(*http2.PushPromiseFrame) + directFramer.WritePushPromise(http2.PushPromiseParam{ + StreamID: f.Header().StreamID, + PromiseID: fr.PromiseID, + BlockFragment: fr.HeaderBlockFragment(), + EndHeaders: fr.HeadersEnded(), + PadLength: 0, + }) + case http2.FramePing: + fr := f.(*http2.PingFrame) + directFramer.WritePing(fr.IsAck(), fr.Data) + case http2.FrameGoAway: + fr := f.(*http2.GoAwayFrame) + directFramer.WriteGoAway(fr.LastStreamID, fr.ErrCode, fr.DebugData()) + case http2.FrameWindowUpdate: + fr := f.(*http2.WindowUpdateFrame) + directFramer.WriteWindowUpdate(f.Header().StreamID, fr.Increment) + case http2.FrameContinuation: + fr := f.(*http2.ContinuationFrame) + directFramer.WriteContinuation(f.Header().StreamID, fr.HeadersEnded(), fr.HeaderBlockFragment()) + default: + fr := f.(*http2.UnknownFrame) + directFramer.WriteRawFrame(f.Header().Type, f.Header().Flags, f.Header().StreamID, fr.Payload()) + } + + return true +} + +func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpack.Decoder) ([]hpack.HeaderField, []byte) { + buf := new(bytes.Buffer) + res := make([]hpack.HeaderField, 0) + + hostIndex := 0 + pathIndex := 0 + decoder.SetEmitEnabled(true) + decoder.SetMaxStringLength(16 << 20) + decoder.SetEmitFunc(func(hf hpack.HeaderField) { + if len(hf.Name) > 0 { + if hf.Name == ":path" { + pathIndex = len(res) + } else if hf.Name == ":authority" { + hostIndex = len(res) + } + res = append(res, hf) + } + }) + defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) + defer decoder.Close() + + buf.Write(fr.HeaderBlockFragment()) + _, err := decoder.Write(fr.HeaderBlockFragment()) + if err != nil { + log.Printf("Error decode %v", err) + } + if fr.HeadersEnded() { + if hostIndex > 0 || pathIndex > 0 { + res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + } + + return res, buf.Bytes() + } + for { + if f, err := framer.ReadFrame(); err != nil { + break + } else { + continuationFrame := f.(*http2.ContinuationFrame) // guaranteed by checkFrameOrder + buf.Write(continuationFrame.HeaderBlockFragment()) + _, err = decoder.Write(continuationFrame.HeaderBlockFragment()) + if err != nil { + log.Printf("Error decode %v", err) + } + if continuationFrame.HeadersEnded() { + break + } + } + } + + if hostIndex > 0 || pathIndex > 0 { + res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + } + + return res, buf.Bytes() +} + +func writeHeaders(framer *http2.Framer, param *http2.HeadersFrameParam, decoder *hpack.Decoder) { + dataToSend := param.BlockFragment + chunkSize := 15 * 1024 + for i := 0; i < len(dataToSend); i += chunkSize { + end := i + chunkSize + headesEnd := false + if end >= len(dataToSend) { + end = len(dataToSend) + headesEnd = true + } + + if i == 0 { + /* decoder.SetEmitEnabled(true) + decoder.SetMaxStringLength(16 << 20) + decoder.SetEmitFunc(func(hf hpack.HeaderField) { + if len(hf.Name) > 0 { + log.Printf("Writing header id:%d, %s:%s", param.StreamID, hf.Name, hf.Value) + } + }) + defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) + defer decoder.Close() + + decoder.Write(dataToSend[i:end])*/ + + framer.WriteHeaders(http2.HeadersFrameParam{ + StreamID: param.StreamID, + BlockFragment: dataToSend[i:end], + EndStream: headesEnd && param.EndStream, + EndHeaders: headesEnd, + PadLength: 0, + Priority: param.Priority, + }) + } else { + framer.WriteContinuation(param.StreamID, headesEnd, dataToSend[i:end]) + } + } +} + +func makeHttpRequest(body []byte, header []hpack.HeaderField) *http.Request { + req := http.Request{} + req.Proto = "http/2" + req.ProtoMajor = 2 + req.ProtoMinor = 0 + + req.Header = http.Header{} + scheme := "https" + authority := "" + path := "" + for _, v := range header { + if !strings.HasPrefix(v.Name, ":") { + req.Header.Add(v.Name, v.Value) + } else if v.Name == ":scheme" { + scheme = v.Value + } else if v.Name == ":authority" { + authority = v.Value + } else if v.Name == ":path" { + path = v.Value + } else if v.Name == ":method" { + req.Method = v.Value + } + } + + req.RequestURI = scheme + "://" + authority + path + req.URL, _ = url.ParseRequestURI(req.RequestURI) + req.Host = req.URL.Host + + if len(body) == 0 { + req.Body = http.NoBody + } else { + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + } + return &req +} + +func makeHttpResponse(body []byte, header []hpack.HeaderField) *http.Response { + resp := http.Response{} + + resp.Proto = "http/2" + resp.ProtoMajor = 2 + resp.ProtoMinor = 0 + resp.Header = http.Header{} + for _, v := range header { + if !strings.HasPrefix(v.Name, ":") { + resp.Header.Add(v.Name, v.Value) + } else if v.Name == ":status" { + resp.Status = v.Value + resp.StatusCode, _ = strconv.Atoi(v.Value) + } + } + if len(body) == 0 { + resp.Body = http.NoBody + resp.ContentLength = 0 + } else { + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + } + return &resp +} + +func putResponseBody(bodyParts [][]byte, resp *http.Response) { + if len(bodyParts) == 0 { + resp.Body = http.NoBody + } else { + body := make([]byte, 0) + for _, b := range bodyParts { + body = append(body, b...) + } + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + } +} + +func encodeHeaderFields(fields []hpack.HeaderField) []byte { + buf := new(bytes.Buffer) + encoder := hpack.NewEncoder(buf) + encoder.SetMaxDynamicTableSizeLimit(65536) + buf.Reset() + + for i := 0; i < len(fields); i++ { + encoder.WriteField(fields[i]) + } + return buf.Bytes() +} + +func encodeHeaders(resp *http.Response) []byte { + buf := new(bytes.Buffer) + encoder := hpack.NewEncoder(buf) + // encoder.SetMaxDynamicTableSize(65536) + buf.Reset() + + writeHeader(encoder, ":status", strconv.Itoa(resp.StatusCode)) + for k, vv := range resp.Header { + lowKey := strings.ToLower(k) + for _, v := range vv { + writeHeader(encoder, lowKey, v) + } + } + return buf.Bytes() +} + +func writeHeader(encoder *hpack.Encoder, name, value string) { + encoder.WriteField(hpack.HeaderField{Name: name, Value: value}) +} diff --git a/goproxy/main.go b/goproxy/main.go index 4c61a4e..0264d5f 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -109,6 +109,7 @@ func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char) int16 { debug.SetTraceback("all") debug.SetPanicOnFault(true) + initIpUtil() if !checkPortAvailable(portHttp) || !checkPortAvailable(portHttps) { return ERROR_PORTS_BUSY @@ -134,6 +135,21 @@ func StopGoServer() { stopGoProxyServer() } +//export IsIpPrivate +func IsIpPrivate(ipStringC *C.char) int16 { + ipString := C.GoString(ipStringC) + ip := net.ParseIP(ipString) + if ip == nil { + log.Printf("Error parsing ip address %s", ipString) + return 0 + } + + if isPrivateIP(ip) { + return 1 + } + return 0 +} + func main() { test() } diff --git a/goproxy/request.go b/goproxy/request.go index 711efd6..e50fa53 100644 --- a/goproxy/request.go +++ b/goproxy/request.go @@ -127,3 +127,18 @@ func RequestGetHeaders(id int64, keys *string) int { return len(request.Header) } + +//returns new path +func HostPathForceSafeSearch(host, path string) string { + // enforce Google safe-search + if strings.Contains(host, "google.com") && strings.Contains(path, "/search?") && !strings.Contains(path, "safe=active") { + return strings.Replace(path+"&safe=active", "&safe=images", "", -1) + // enforce Bing safe-search + } else if strings.Contains(host, "bing.com") && strings.Contains(path, "/search?") && !strings.Contains(path, "adlt=strict") { + return path + "&adlt=strict" + // enforce Yahoo safe-search + } else if strings.Contains(host, "yahoo.com") && strings.Contains(path, "/search?") && !strings.Contains(path, "&vm=r") { + return path + "&vm=r" + } + return path +} From 53319cf72a7d2ada65e202b611ac10f9684c4091 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Mon, 10 May 2021 17:55:15 +0200 Subject: [PATCH 15/27] one more fix --- goproxy/goproxystarter.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 39aebe3..0dc8838 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -17,6 +17,7 @@ import ( "bufio" "bytes" "context" + "crypto/tls" "encoding/base64" "encoding/binary" "fmt" @@ -75,6 +76,19 @@ func initGoProxy() { proxy.ServeHTTP(w, req) }) + proxy.Tr = &http.Transport{ + MaxIdleConnsPerHost: 10, + MaxIdleConns: 1000, + IdleConnTimeout: time.Minute * 10, + TLSClientConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + InsecureSkipVerify: true, + CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256}, + PreferServerCipherSuites: true, + Renegotiation: tls.RenegotiateFreelyAsClient, + }, + } + proxy.OnRequest().HandleConnect(handleConnectFunc) if proxy.Verbose { From f1e02d3dbd2de4e855e05264ab835c0b41b36392 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Tue, 21 Sep 2021 17:10:31 +0700 Subject: [PATCH 16/27] http/2 protocol issue fix --- goproxy/adblock.go | 2 +- goproxy/build-internal.bat | 28 ------ goproxy/build-internal.sh | 28 ------ goproxy/build.bat | 28 ------ goproxy/build.sh | 20 ++--- goproxy/goproxystarter.go | 7 ++ goproxy/http2.go | 173 ++++++++++++++++++++++++------------- goproxy/request.go | 1 - 8 files changed, 132 insertions(+), 155 deletions(-) delete mode 100644 goproxy/build-internal.bat delete mode 100644 goproxy/build-internal.sh delete mode 100644 goproxy/build.bat mode change 100644 => 100755 goproxy/build.sh diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 664fddb..0470b61 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -175,7 +175,7 @@ func (am *AdBlockMatcher) IsDomainWhitelisted(host string) bool { } func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string) (category *string, matchType int, isRelaxedPolicy bool) { - if am.RulesCnt == 0 { + if am == nil || am.RulesCnt == 0 { return nil, Included, false } diff --git a/goproxy/build-internal.bat b/goproxy/build-internal.bat deleted file mode 100644 index e5e1ea0..0000000 --- a/goproxy/build-internal.bat +++ /dev/null @@ -1,28 +0,0 @@ -@echo off - -REM this file is for setting up individual builds for x64 vs x86. -REM specify make directories in build.bat -REM Needs: -REM %GOPROXY_BIN% -REM %GOPROXY_GOROOT% -REM %GOPROXY_PATH% - -SET BUILDMODE=c-shared - -IF "%PROXY_OUTPUT_FILE%"=="" SET PROXY_OUTPUT_FILE=proxy.dll -IF "%PROXY_OUTPUT_FILE%"=="proxy.exe" SET BUILDMODE=exe - -set OLD_GOROOT=%GOROOT% -set OLD_GOPATH=%GOPATH% - -set GOROOT=%GOPROXY_GOROOT% -set GOPATH=%GOPROXY_GOPATH% - -%GOPROXY_PATH%\go get -d .\... - -mkdir %GOPROXY_BIN% - -%GOPROXY_PATH%\go build -ldflags "-s -w" --buildmode=%BUILDMODE% -o %GOPROXY_BIN%\%PROXY_OUTPUT_FILE% - -set GOROOT=%OLD_GOROOT% -set GOPATH=%OLD_GOPATH% diff --git a/goproxy/build-internal.sh b/goproxy/build-internal.sh deleted file mode 100644 index 5ef7d67..0000000 --- a/goproxy/build-internal.sh +++ /dev/null @@ -1,28 +0,0 @@ - -# this file is for setting up individual builds for x64 vs x86. -# specify make directories in build.bat -# Needs: -# $GOPROXY_BIN -# $GOPROXY_GOROOT -# $GOPROXY_PATH - -export BUILDMODE=c-shared - -if [ "$PROXY_OUTPUT_FILE" == "" ]; then - export PROXY_OUTPUT_FILE=libproxy.so -fi; - -if [ "$PROXY_OUTPUT_FILE" == "libproxy.so" ] || [ "$PROXY_OUTPUT_FILE" == "libproxy.dylib" ] || [ "$PROXY_OUTPUT_FILE" == "proxy.dll" ]; then - export BUILDMODE=c-shared -else - export BUILDMODE=exe -fi; - -OLD_GOROOT=$GOROOT -OLD_GOPATH=$GOPATH - -go get -d ./... - -mkdir -p $GOPROXY_BIN - -go build -ldflags "-s -w" --buildmode=$BUILDMODE -o $GOPROXY_BIN/$PROXY_OUTPUT_FILE diff --git a/goproxy/build.bat b/goproxy/build.bat deleted file mode 100644 index 75c736a..0000000 --- a/goproxy/build.bat +++ /dev/null @@ -1,28 +0,0 @@ -set PROXY_OUTPUT_FILE=proxy.dll -set OLD_PATH=%PATH% - -set BASE_PATH=%SystemRoot%;%SystemRoot%\System32 - -set PATH=%BASE_PATH%;C:\Go\bin;C:\msys64\mingw64\bin;C:\Program Files\Git\bin - -set GOARCH=amd64 -set GOPROXY_BIN=bin\x64 -set GOPROXY_GOROOT=C:\Go -set GOPROXY_PATH=C:\Go\bin -set GOPROXY_GOPATH=%UserProfile%\go -set CGO_LDFLAGS= -set CC=C:\msys64\mingw64\bin\gcc -set CGO_ENABLED=1 -call build-internal.bat - -REM TODO Add Darwin building - -set PATH=%BASE_PATH%;C:\Go\bin;C:\msys64\mingw32\bin;C:\Program Files\Git\bin -set GOARCH=386 -set GOPROXY_BIN=bin\x86 -set GOPROXY_GOPATH=%UserProfile%\go32 -set CC=C:\msys64\mingw32\bin\gcc -call build-internal.bat - -set PATH=%OLD_PATH% - diff --git a/goproxy/build.sh b/goproxy/build.sh old mode 100644 new mode 100755 index ae60cb1..c86295d --- a/goproxy/build.sh +++ b/goproxy/build.sh @@ -2,21 +2,19 @@ export GOPROXY_BIN=$PWD/bin/x64 -export PROXY_OUTPUT_FILE=libproxy.dylib - export GOOS=darwin export GOARCH=amd64 export CC=gcc export CGO_ENABLED=1 -bash build-internal.sh -export PROXY_OUTPUT_FILE=proxy.dll -export GOOS=windows -export CC=x86_64-w64-mingw32-gcc -bash build-internal.sh +echo "Building x64..." +go build -ldflags "-s -w" --buildmode=c-archive -o proxy-x64.a + +echo "Building arm..." +export GOARCH=arm64 +go build -ldflags "-s -w" --buildmode=c-archive -o proxy-arm64.a -export GOPROXY_BIN=$PWD/bin/x86 -export GOARCH=386 -export CC=i686-w64-mingw32-gcc -bash build-internal.sh +lipo -create proxy-x64.a proxy-arm64.a -output proxy.a +lipo -info proxy.a +echo "done" \ No newline at end of file diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 0dc8838..ca64bb6 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -239,6 +239,9 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert proxy.OnRequest().DoFunc( func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + userData := make(map[string]interface{}) + ctx.UserData = userData + monitorLogFileSize() if adBlockMatcher != nil { category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(r.URL.String(), r.Host, r.Referer()) @@ -260,6 +263,10 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy)) } } + + // if strings.Contains(r.Host, "yandex") { + // return r, goproxy.NewResponse(r, "text/html; charset=UTF-8", 200, "Blocked by rules1") + // } return r, nil }) diff --git a/goproxy/http2.go b/goproxy/http2.go index 373ddd9..516c84f 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -11,7 +11,9 @@ import ( "strconv" "strings" "sync/atomic" + "time" + "github.com/bep/debounce" "github.com/cloudveiltech/goproxy" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" @@ -19,27 +21,32 @@ import ( var http2ProxySessionCounter int64 +const MAX_FILTERABLE_LENGTH = 1024 * 1024 const MIN_FILTERABLE_LENGTH = 100 type Http2Handler struct { - lastHttpResponse map[uint32]*http.Response - lastHttpRequest map[uint32]*http.Request - lastHeadersBlock map[uint32]*http2.HeadersFrameParam - proxyCtx map[uint32]*goproxy.ProxyCtx - lastHeadersMap map[uint32][]hpack.HeaderField - responseBodyMapChunks map[uint32][][]byte + lastHttpResponse map[uint32]*http.Response + lastHttpRequest map[uint32]*http.Request + lastHeadersBlock map[uint32]*http2.HeadersFrameParam + proxyCtx map[uint32]*goproxy.ProxyCtx + lastHeadersMap map[uint32][]hpack.HeaderField + responseBodyMapChunks map[uint32][][]byte + debouncers map[uint32]func(f func()) + connectionReadyForData bool } func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Conn) bool { log.Print("Running http2 handler for " + r.URL.String()) http2Handler := &Http2Handler{ - lastHttpResponse: make(map[uint32]*http.Response), - lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), - lastHeadersMap: make(map[uint32][]hpack.HeaderField), - lastHttpRequest: make(map[uint32]*http.Request), - proxyCtx: make(map[uint32]*goproxy.ProxyCtx), - responseBodyMapChunks: make(map[uint32][][]byte), + lastHttpResponse: make(map[uint32]*http.Response), + lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), + lastHeadersMap: make(map[uint32][]hpack.HeaderField), + lastHttpRequest: make(map[uint32]*http.Request), + proxyCtx: make(map[uint32]*goproxy.ProxyCtx), + responseBodyMapChunks: make(map[uint32][][]byte), + debouncers: make(map[uint32]func(f func())), + connectionReadyForData: false, } go func() { http2Handler.processHttp2Stream(rawClientTls, remote) @@ -101,64 +108,103 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F fr := f.(*http2.DataFrame) body := fr.Data() - lastHttpResponse := http2Handler.lastHttpResponse[f.Header().StreamID] - bodyChunks := http2Handler.responseBodyMapChunks[f.Header().StreamID] + streamId := f.Header().StreamID + lastHttpResponse := http2Handler.lastHttpResponse[streamId] + bodyChunks := http2Handler.responseBodyMapChunks[streamId] chunk := make([]byte, len(body)) copy(chunk, body) bodyChunks = append(bodyChunks, chunk) - http2Handler.responseBodyMapChunks[f.Header().StreamID] = bodyChunks + http2Handler.responseBodyMapChunks[streamId] = bodyChunks + + ctx := http2Handler.proxyCtx[streamId] + + blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] + whitelisted := exists && !(blocked.(bool)) + + processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { + if force { + log.Print("Force ending stream on timeout") + } + lastHttpResponse = http2Handler.lastHttpResponse[streamId] + bodyChunks = http2Handler.responseBodyMapChunks[streamId] + + if lastHttpResponse != nil { + log.Printf("Process frame data %s", lastHttpResponse.Request.RequestURI) + } + + streamEnded := fr.StreamEnded() || force + if !whitelisted && lastHttpResponse != nil && !client { + contentType := lastHttpResponse.Header.Get("Content-Type") + isContentTypeFilterable := isContentTypeFilterable(contentType) - if lastHttpResponse != nil && !client { - contentType := lastHttpResponse.Header.Get("Content-Type") - isContentTypeFilterable := isContentTypeFilterable(contentType) - if isContentTypeFilterable && fr.StreamEnded() { putResponseBody(bodyChunks, lastHttpResponse) contentLength := lastHttpResponse.ContentLength - if contentLength > MIN_FILTERABLE_LENGTH { - ctx := http2Handler.proxyCtx[f.Header().StreamID] - resp := proxy.FilterResponse(lastHttpResponse, ctx) - - if resp != lastHttpResponse { //new response - writeHeaders(directFramer, &http2.HeadersFrameParam{ - StreamID: f.Header().StreamID, - BlockFragment: encodeHeaders(resp), - EndStream: false, - EndHeaders: true, - PadLength: 0, - Priority: http2.PriorityParam{}, - }, decoder) - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - directFramer.WriteData(f.Header().StreamID, true, buf.Bytes()) - directFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeCancel, nil) - delete(http2Handler.lastHttpResponse, f.Header().StreamID) - delete(http2Handler.lastHttpRequest, f.Header().StreamID) - delete(http2Handler.responseBodyMapChunks, f.Header().StreamID) - return false + isContentTypeFilterable = isContentTypeFilterable && contentLength < MAX_FILTERABLE_LENGTH + if isContentTypeFilterable && streamEnded { + if contentLength > MIN_FILTERABLE_LENGTH { + ctx := http2Handler.proxyCtx[streamId] + resp := proxy.FilterResponse(lastHttpResponse, ctx) + + if resp != lastHttpResponse { //new response + if !http2Handler.connectionReadyForData { + reverseFramer.WriteSettings() + } + writeHeaders(directFramer, &http2.HeadersFrameParam{ + StreamID: streamId, + BlockFragment: encodeHeaders(resp), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: http2.PriorityParam{}, + }, decoder) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + directFramer.WriteData(streamId, true, buf.Bytes()) + directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) + delete(http2Handler.lastHttpResponse, streamId) + delete(http2Handler.lastHttpRequest, streamId) + delete(http2Handler.responseBodyMapChunks, streamId) + return false + } } + } else if isContentTypeFilterable { + return true } - } else if isContentTypeFilterable { - return true } - } - header, ok := http2Handler.lastHeadersBlock[f.Header().StreamID] - if ok { - // headerFields, _ := http2Handler.lastHeadersMap[f.Header().StreamID] - header.EndStream = false - // header.BlockFragment = encodeHeaderFields(headerFields) - writeHeaders(directFramer, header, decoder) - delete(http2Handler.lastHeadersBlock, f.Header().StreamID) - delete(http2Handler.lastHeadersMap, f.Header().StreamID) - } + header, ok := http2Handler.lastHeadersBlock[streamId] + if ok { + // headerFields, _ := http2Handler.lastHeadersMap[streamId] + header.EndStream = false + // header.BlockFragment = encodeHeaderFields(headerFields) + writeHeaders(directFramer, header, decoder) + delete(http2Handler.lastHeadersBlock, streamId) + delete(http2Handler.lastHeadersMap, streamId) + } + + for i, _ := range bodyChunks { + streamEnd := i == len(bodyChunks)-1 && streamEnded + directFramer.WriteData(streamId, streamEnd, bodyChunks[i]) + } - for i, _ := range bodyChunks { - streamEnded := i == len(bodyChunks)-1 && fr.StreamEnded() - directFramer.WriteData(f.Header().StreamID, streamEnded, bodyChunks[i]) + delete(http2Handler.responseBodyMapChunks, streamId) + return true } - delete(http2Handler.responseBodyMapChunks, f.Header().StreamID) + processDataFrameFunc(false, streamId, directFramer, reverseFramer, decoder, client) + + debouncer, exists := http2Handler.debouncers[streamId] + if !exists { + debouncer = debounce.New(time.Millisecond * 1000) + http2Handler.debouncers[streamId] = debouncer + } + debouncer(func() { + _, exists := http2Handler.debouncers[streamId] + if exists { + processDataFrameFunc(true, streamId, directFramer, reverseFramer, decoder, client) + } + }) case http2.FrameHeaders: fr := f.(*http2.HeadersFrame) @@ -173,7 +219,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F http2Handler.lastHttpRequest[f.Header().StreamID] = request http2Handler.proxyCtx[f.Header().StreamID] = ctx _, resp := proxy.FilterRequest(request, ctx) + if resp != nil { + if !http2Handler.connectionReadyForData { + reverseFramer.WriteSettings() + } writeHeaders(reverseFramer, &http2.HeadersFrameParam{ StreamID: f.Header().StreamID, BlockFragment: encodeHeaders(resp), @@ -185,7 +235,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F buf := new(bytes.Buffer) buf.ReadFrom(resp.Body) reverseFramer.WriteData(f.Header().StreamID, true, buf.Bytes()) - reverseFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeCancel, nil) + reverseFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeRefusedStream, nil) return false } } else { @@ -222,6 +272,10 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F directFramer.WriteRSTStream(f.Header().StreamID, fr.ErrCode) case http2.FrameSettings: fr := f.(*http2.SettingsFrame) + if !client { + http2Handler.connectionReadyForData = true //once server sent the settings we're good to go + } + if fr.IsAck() { directFramer.WriteSettingsAck() } else { @@ -229,15 +283,17 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F for i := 0; i < fr.NumSettings(); i++ { setting := fr.Setting(i) params = append(params, setting) - if setting.ID == http2.SettingHeaderTableSize { + if setting.ID == http2.SettingHeaderTableSize && client { decoder.SetMaxDynamicTableSize(setting.Val) } } directFramer.WriteSettings(params...) } + case http2.FramePushPromise: fr := f.(*http2.PushPromiseFrame) directFramer.WritePushPromise(http2.PushPromiseParam{ + StreamID: f.Header().StreamID, PromiseID: fr.PromiseID, BlockFragment: fr.HeaderBlockFragment(), @@ -452,6 +508,7 @@ func encodeHeaders(resp *http.Response) []byte { buf.Reset() writeHeader(encoder, ":status", strconv.Itoa(resp.StatusCode)) + writeHeader(encoder, "content-length", strconv.FormatInt(resp.ContentLength, 10)) for k, vv := range resp.Header { lowKey := strings.ToLower(k) for _, v := range vv { diff --git a/goproxy/request.go b/goproxy/request.go index e50fa53..03feeac 100644 --- a/goproxy/request.go +++ b/goproxy/request.go @@ -128,7 +128,6 @@ func RequestGetHeaders(id int64, keys *string) int { return len(request.Header) } -//returns new path func HostPathForceSafeSearch(host, path string) string { // enforce Google safe-search if strings.Contains(host, "google.com") && strings.Contains(path, "/search?") && !strings.Contains(path, "safe=active") { From 2af2e656fcd710a3c23f616f724f2dee62217305 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Sat, 20 Nov 2021 13:29:01 +0200 Subject: [PATCH 17/27] add image filtering --- goproxy/goproxystarter.go | 39 +++++++++++++++++------ goproxy/http2.go | 66 ++++++++++++++++++++++++++++----------- goproxy/main.go | 29 +++++++++++------ 3 files changed, 96 insertions(+), 38 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index ca64bb6..7162234 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -38,6 +38,10 @@ import ( //import _ "net/http/pprof" +var BLOCKED_IMAGE_BYTES []byte + +const BLOCKED_IMAGE_CONTENT_TYPE = "image/webp" + const DEFAULT_HTTPS_PORT uint16 = 443 type HttpsHandler func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) @@ -297,7 +301,9 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert } } - if !adBlockMatcher.TestContentTypeIsFiltrable(resp.Header.Get("Content-Type")) { + contentType := resp.Header.Get("Content-Type") + isImage := strings.Contains(contentType, "image") + if !adBlockMatcher.TestContentTypeIsFiltrable(contentType) && !isImage { return resp } buf := new(bytes.Buffer) @@ -309,18 +315,31 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert resp.Body.Close() resp.Body = ioutil.NopCloser(bytes.NewBuffer(bytesData)) - if !adBlockMatcher.IsContentSmallEnoughToFilter(int64(len(bytesData))) { - return resp - } + if isImage && len(bytesData) > 10240 { //10kb + err, isAllowed := CheckImage(resp.Request.RequestURI, bytesData) + if err != nil { + log.Printf("Image checking fail: %v", err) + } else { + if isAllowed { + return resp + } else { + return goproxy.NewResponse(resp.Request, BLOCKED_IMAGE_CONTENT_TYPE, 200, string(BLOCKED_IMAGE_BYTES)) + } + } + } else { + if !adBlockMatcher.IsContentSmallEnoughToFilter(int64(len(bytesData))) { + return resp + } - bytesData = decodeResponseCompression(resp.Header.Get("Content-Encoding"), bytesData) + bytesData = decodeResponseCompression(resp.Header.Get("Content-Encoding"), bytesData) - category, matches := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) + category, matches := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) - if category != nil { - log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", resp.Request.URL.String(), *category, strings.Join(matches, ", ")) - message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false) - return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + if category != nil { + log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", resp.Request.URL.String(), *category, strings.Join(matches, ", ")) + message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false) + return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + } } return resp }) diff --git a/goproxy/http2.go b/goproxy/http2.go index 516c84f..9fa058f 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -11,9 +11,7 @@ import ( "strconv" "strings" "sync/atomic" - "time" - "github.com/bep/debounce" "github.com/cloudveiltech/goproxy" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" @@ -25,6 +23,7 @@ const MAX_FILTERABLE_LENGTH = 1024 * 1024 const MIN_FILTERABLE_LENGTH = 100 type Http2Handler struct { + maxFrameSize uint32 lastHttpResponse map[uint32]*http.Response lastHttpRequest map[uint32]*http.Request lastHeadersBlock map[uint32]*http2.HeadersFrameParam @@ -39,6 +38,7 @@ func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Co log.Print("Running http2 handler for " + r.URL.String()) http2Handler := &Http2Handler{ + maxFrameSize: 1024, lastHttpResponse: make(map[uint32]*http.Response), lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), lastHeadersMap: make(map[uint32][]hpack.HeaderField), @@ -92,8 +92,15 @@ func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tl } func isContentTypeFilterable(contentType string) bool { + if strings.Contains(contentType, "protobuf") { + return false + } return strings.Contains(contentType, "html") || - strings.Contains(contentType, "json") + strings.Contains(contentType, "json") || + strings.Contains(contentType, "image/png") || + strings.Contains(contentType, "image/jpg") || + strings.Contains(contentType, "image/jpeg") || + strings.Contains(contentType, "image/webp") } func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { @@ -136,11 +143,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if !whitelisted && lastHttpResponse != nil && !client { contentType := lastHttpResponse.Header.Get("Content-Type") isContentTypeFilterable := isContentTypeFilterable(contentType) - + isImage := strings.Contains(contentType, "image") putResponseBody(bodyChunks, lastHttpResponse) contentLength := lastHttpResponse.ContentLength - isContentTypeFilterable = isContentTypeFilterable && contentLength < MAX_FILTERABLE_LENGTH + isContentTypeFilterable = isContentTypeFilterable && (contentLength < MAX_FILTERABLE_LENGTH || isImage) if isContentTypeFilterable && streamEnded { if contentLength > MIN_FILTERABLE_LENGTH { ctx := http2Handler.proxyCtx[streamId] @@ -160,8 +167,8 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F }, decoder) buf := new(bytes.Buffer) buf.ReadFrom(resp.Body) - directFramer.WriteData(streamId, true, buf.Bytes()) - directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) + writeFinalData(directFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) + // directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) delete(http2Handler.lastHttpResponse, streamId) delete(http2Handler.lastHttpRequest, streamId) delete(http2Handler.responseBodyMapChunks, streamId) @@ -193,18 +200,18 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F } processDataFrameFunc(false, streamId, directFramer, reverseFramer, decoder, client) - - debouncer, exists := http2Handler.debouncers[streamId] - if !exists { - debouncer = debounce.New(time.Millisecond * 1000) - http2Handler.debouncers[streamId] = debouncer - } - debouncer(func() { - _, exists := http2Handler.debouncers[streamId] - if exists { - processDataFrameFunc(true, streamId, directFramer, reverseFramer, decoder, client) + /* + debouncer, exists := http2Handler.debouncers[streamId] + if !exists { + debouncer = debounce.New(time.Millisecond * 1000) + http2Handler.debouncers[streamId] = debouncer } - }) + debouncer(func() { + _, exists := http2Handler.debouncers[streamId] + if exists { + processDataFrameFunc(true, streamId, directFramer, reverseFramer, decoder, client) + } + })*/ case http2.FrameHeaders: fr := f.(*http2.HeadersFrame) @@ -234,7 +241,8 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F }, decoder) buf := new(bytes.Buffer) buf.ReadFrom(resp.Body) - reverseFramer.WriteData(f.Header().StreamID, true, buf.Bytes()) + + writeFinalData(reverseFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) reverseFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeRefusedStream, nil) return false } @@ -286,6 +294,12 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if setting.ID == http2.SettingHeaderTableSize && client { decoder.SetMaxDynamicTableSize(setting.Val) } + if setting.ID == http2.SettingMaxFrameSize && client { + if http2Handler.maxFrameSize > setting.Val { + http2Handler.maxFrameSize = setting.Val + } + } + } directFramer.WriteSettings(params...) } @@ -376,6 +390,20 @@ func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpa return res, buf.Bytes() } +func writeFinalData(framer *http2.Framer, streamId uint32, data *bytes.Buffer, chunkSize int) { + dataToSend := data.Bytes() + for i := 0; i < len(dataToSend); i += chunkSize { + end := i + chunkSize + dataEnded := false + if end >= len(dataToSend) { + end = len(dataToSend) + dataEnded = true + } + + framer.WriteData(streamId, dataEnded, dataToSend[i:end]) + } +} + func writeHeaders(framer *http2.Framer, param *http2.HeadersFrameParam, decoder *hpack.Decoder) { dataToSend := param.BlockFragment chunkSize := 15 * 1024 diff --git a/goproxy/main.go b/goproxy/main.go index 0264d5f..a3baf19 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -5,6 +5,7 @@ import "C" import ( "fmt" "io" + "io/ioutil" "log" "net" "os" @@ -16,6 +17,7 @@ const ( SUCCESS = 1 ERROR_PORTS_BUSY = -1 ERROR_CERTS_GENERATION = -2 + ERROR_IMAGE_READ = -3 MAX_LOG_SIZE = 10 * 1024 * 1024 ) @@ -106,7 +108,7 @@ func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { } //export StartGoServer -func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char) int16 { +func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char, bannedImageFileC *C.char) int16 { debug.SetTraceback("all") debug.SetPanicOnFault(true) initIpUtil() @@ -125,6 +127,12 @@ func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC } } + bannedImagePath := C.GoString(bannedImageFileC) + BLOCKED_IMAGE_BYTES, err = ioutil.ReadFile(bannedImagePath) + if err != nil { + log.Printf("Can't load blocked image: %v", err) + return ERROR_IMAGE_READ + } startGoProxyServer(portHttp, portHttps, portConfigurationServer, certFile, keyFile) monitorLogFileSize() return SUCCESS @@ -155,19 +163,22 @@ func main() { } func test() { - log.Printf("main: starting HTTP server") + file, _ := ioutil.ReadFile("/Users/dgoraschenko/Downloads/4.png") + CheckImage("testurl", file) + + /*log.Printf("main: starting HTTP server") startGoProxyServer(14600, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") log.Printf("main: serving for 1000 seconds") var quit = false - - for !quit { - //line, _ = reader.ReadString('\n') - //if strings.TrimSpace(line) == "quit" { - // quit = true - //} - } + */ + // for !quit { + //line, _ = reader.ReadString('\n') + //if strings.TrimSpace(line) == "quit" { + // quit = true + //} + // } // Stop() // log.Printf("main: done. exiting") From ae836228b94e73d5b30c8bafae68189918948c47 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 25 Nov 2021 16:12:00 +0200 Subject: [PATCH 18/27] crash fix --- goproxy/goproxystarter.go | 2 +- goproxy/http2.go | 9 ++++++--- goproxy/main.go | 19 +------------------ 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 7162234..7b4430b 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -345,7 +345,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert }) runConfigurationServerListener() - go runHttpsListener() + runHttpsListener() if proxy.Verbose { log.Printf("Server started") diff --git a/goproxy/http2.go b/goproxy/http2.go index 9fa058f..cccc20b 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -125,8 +125,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F ctx := http2Handler.proxyCtx[streamId] - blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] - whitelisted := exists && !(blocked.(bool)) + whitelisted := false + if ctx != nil && ctx.UserData != nil { + blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] + whitelisted = exists && !(blocked.(bool)) + } processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { if force { @@ -135,7 +138,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F lastHttpResponse = http2Handler.lastHttpResponse[streamId] bodyChunks = http2Handler.responseBodyMapChunks[streamId] - if lastHttpResponse != nil { + if lastHttpResponse != nil && lastHttpResponse.Request != nil { log.Printf("Process frame data %s", lastHttpResponse.Request.RequestURI) } diff --git a/goproxy/main.go b/goproxy/main.go index a3baf19..0b95be1 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -163,23 +163,6 @@ func main() { } func test() { - file, _ := ioutil.ReadFile("/Users/dgoraschenko/Downloads/4.png") - CheckImage("testurl", file) - - /*log.Printf("main: starting HTTP server") + log.Printf("main: starting HTTP server") startGoProxyServer(14600, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") - - log.Printf("main: serving for 1000 seconds") - - var quit = false - */ - // for !quit { - //line, _ = reader.ReadString('\n') - //if strings.TrimSpace(line) == "quit" { - // quit = true - //} - // } - - // Stop() - // log.Printf("main: done. exiting") } From 2e89f800f14500cbcf742629282490229c76aa2b Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Tue, 21 Dec 2021 09:38:35 +0100 Subject: [PATCH 19/27] bugfix --- goproxy/goproxystarter.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 7b4430b..bbc228c 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -41,6 +41,7 @@ import ( var BLOCKED_IMAGE_BYTES []byte const BLOCKED_IMAGE_CONTENT_TYPE = "image/webp" +const MIN_FILTERABLE_LENGTH_IMAGE = 1024 const DEFAULT_HTTPS_PORT uint16 = 443 @@ -81,9 +82,10 @@ func initGoProxy() { }) proxy.Tr = &http.Transport{ - MaxIdleConnsPerHost: 10, - MaxIdleConns: 1000, - IdleConnTimeout: time.Minute * 10, + MaxIdleConnsPerHost: 10, + MaxIdleConns: 1000, + IdleConnTimeout: time.Minute * 10, + ResponseHeaderTimeout: time.Minute * 10, TLSClientConfig: &tls.Config{ NextProtos: []string{"http/1.1"}, InsecureSkipVerify: true, @@ -315,7 +317,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert resp.Body.Close() resp.Body = ioutil.NopCloser(bytes.NewBuffer(bytesData)) - if isImage && len(bytesData) > 10240 { //10kb + if isImage && len(bytesData) > MIN_FILTERABLE_LENGTH_IMAGE { err, isAllowed := CheckImage(resp.Request.RequestURI, bytesData) if err != nil { log.Printf("Image checking fail: %v", err) @@ -345,7 +347,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert }) runConfigurationServerListener() - runHttpsListener() + go runHttpsListener() if proxy.Verbose { log.Printf("Server started") From 2adc162fdd7023c62f2595a2261661d4a37ce9fb Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 23 Dec 2021 11:18:33 +0100 Subject: [PATCH 20/27] filtering disabling support --- goproxy/build.sh | 3 ++- goproxy/goproxystarter.go | 3 ++- goproxy/http2.go | 16 ++++++++++------ goproxy/main.go | 17 +++++++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/goproxy/build.sh b/goproxy/build.sh index c86295d..38bac43 100755 --- a/goproxy/build.sh +++ b/goproxy/build.sh @@ -17,4 +17,5 @@ go build -ldflags "-s -w" --buildmode=c-archive -o proxy-arm64.a lipo -create proxy-x64.a proxy-arm64.a -output proxy.a lipo -info proxy.a -echo "done" \ No newline at end of file +cp -f proxy-x64.h proxy.h +echo "done" diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index bbc228c..18993c1 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -304,7 +304,8 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert } contentType := resp.Header.Get("Content-Type") - isImage := strings.Contains(contentType, "image") + isImage := strings.Contains(contentType, "image") && isImageFilteringEnabled + if !adBlockMatcher.TestContentTypeIsFiltrable(contentType) && !isImage { return resp } diff --git a/goproxy/http2.go b/goproxy/http2.go index cccc20b..fb84fef 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -95,12 +95,16 @@ func isContentTypeFilterable(contentType string) bool { if strings.Contains(contentType, "protobuf") { return false } - return strings.Contains(contentType, "html") || - strings.Contains(contentType, "json") || - strings.Contains(contentType, "image/png") || - strings.Contains(contentType, "image/jpg") || - strings.Contains(contentType, "image/jpeg") || - strings.Contains(contentType, "image/webp") + + filterable := strings.Contains(contentType, "html") || strings.Contains(contentType, "json") + + if isImageFilteringEnabled && !filterable { + filterable = strings.Contains(contentType, "image/png") || + strings.Contains(contentType, "image/jpg") || + strings.Contains(contentType, "image/jpeg") || + strings.Contains(contentType, "image/webp") + } + return filterable } func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { diff --git a/goproxy/main.go b/goproxy/main.go index 0b95be1..39edcf2 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -24,6 +24,7 @@ const ( var certsException = make(map[string]bool) var logFilePath = "" var logFileHandle *os.File +var isImageFilteringEnabled = false //export AddCertException func AddCertException(thumbPrintC *C.char) { @@ -128,16 +129,24 @@ func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC } bannedImagePath := C.GoString(bannedImageFileC) - BLOCKED_IMAGE_BYTES, err = ioutil.ReadFile(bannedImagePath) - if err != nil { - log.Printf("Can't load blocked image: %v", err) - return ERROR_IMAGE_READ + if bannedImagePath != "" { + BLOCKED_IMAGE_BYTES, err = ioutil.ReadFile(bannedImagePath) + if err != nil { + log.Printf("Can't load blocked image: %v", err) + return ERROR_IMAGE_READ + } } startGoProxyServer(portHttp, portHttps, portConfigurationServer, certFile, keyFile) monitorLogFileSize() return SUCCESS } +//export SetImageFilteringEnabled +func SetImageFilteringEnabled(enabled bool) { + log.Printf("Setting Image filtering to: %v", enabled) + isImageFilteringEnabled = enabled +} + //export StopGoServer func StopGoServer() { stopGoProxyServer() From a474ccfbe5da2fb3e5262bbf6b752eca16fc5602 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Wed, 1 Feb 2023 19:43:12 +0530 Subject: [PATCH 21/27] crash fixes case sensitive changes --- goproxy/adblock.go | 26 +++--- goproxy/adblock_import.go | 2 + goproxy/adblock_interop.go | 98 ++++++++++++++++------ goproxy/certs.go | 46 +++++----- goproxy/debug.windows.go | 3 +- goproxy/goproxy_chain.go | 35 ++++---- goproxy/goproxystarter.go | 16 +++- goproxy/http2.go | 68 +++++++++++---- goproxy/main.go | 24 ++++++ goproxy/request.go | 138 ++++-------------------------- goproxy/response.go | 166 ------------------------------------- 11 files changed, 238 insertions(+), 384 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 0470b61..2d6ea24 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -71,22 +71,22 @@ type AdBlockMatcher struct { } func CreateMatcher() *AdBlockMatcher { - adBlockMatcher = &AdBlockMatcher{ + newMatcher := &AdBlockMatcher{ RulesCnt: 0, defaultBlockPageTags: make(map[string]string), } - return adBlockMatcher + return newMatcher } func (am *AdBlockMatcher) addMatcher(category string, listType int) { matcher := adblock.NewMatcher() var categoryMatcher *MatcherCategory - matcherCategories := adBlockMatcher.BlackListMatcherCategories + matcherCategories := am.BlackListMatcherCategories if listType == BypassList { - matcherCategories = adBlockMatcher.BypassMatcherCategories + matcherCategories = am.BypassMatcherCategories } else if listType == Whitelist { - matcherCategories = adBlockMatcher.WhiteListMatcherCategories + matcherCategories = am.WhiteListMatcherCategories } for _, element := range matcherCategories { @@ -113,8 +113,8 @@ func (am *AdBlockMatcher) addMatcher(category string, listType int) { } categoryMatcher.Matchers = append(categoryMatcher.Matchers, matcher) - adBlockMatcher.lastMatcher = matcher - adBlockMatcher.lastCategory = categoryMatcher + am.lastMatcher = matcher + am.lastCategory = categoryMatcher } func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPolicy bool) string { @@ -179,6 +179,10 @@ func (am *AdBlockMatcher) TestUrlBlocked(url string, host string, referer string return nil, Included, false } + url = strings.ToLower(url) + host = strings.ToLower(host) + referer = strings.ToLower(referer) + res1, res2 := am.matchRulesCategories(am.WhiteListMatcherCategories, url, host, referer) if res1 != nil { return res1, res2, false @@ -322,7 +326,7 @@ func isNonLetterAndDigitRune(r rune) bool { func (am *AdBlockMatcher) AddBlockedPhrase(phrase string, category string) { var phraseCategory *PhraseCategory = nil - for _, element := range adBlockMatcher.PhraseCategories { + for _, element := range am.PhraseCategories { if element.Category == category { phraseCategory = element break @@ -407,14 +411,14 @@ func LoadMatcherFromFile(filePath string) *AdBlockMatcher { decoder := gob.NewDecoder(stream) - adBlockMatcher = &AdBlockMatcher{ + adBlockMatcherLoaded := &AdBlockMatcher{ RulesCnt: 0, } - err = decoder.Decode(&adBlockMatcher) + err = decoder.Decode(&adBlockMatcherLoaded) if err != nil { log.Printf("Decoder error %s", err) } - return adBlockMatcher + return adBlockMatcherLoaded } func (am *AdBlockMatcher) EnableBypass() { diff --git a/goproxy/adblock_import.go b/goproxy/adblock_import.go index fbfdf75..675e757 100644 --- a/goproxy/adblock_import.go +++ b/goproxy/adblock_import.go @@ -24,6 +24,8 @@ func (am *AdBlockMatcher) ParseRulesZipArchive(filePath string) { } func (am *AdBlockMatcher) AddRule(rule string, category string, listType int) { + rule = strings.ToLower(rule) + r, e := adblock.ParseRule(rule) if e != nil { diff --git a/goproxy/adblock_interop.go b/goproxy/adblock_interop.go index 5a5e2ff..ea086cf 100644 --- a/goproxy/adblock_interop.go +++ b/goproxy/adblock_interop.go @@ -8,6 +8,8 @@ import ( "log" "os" "strings" + "sync" + "time" "unsafe" "github.com/aymerick/raymond" @@ -16,33 +18,42 @@ import ( var adBlockBlacklistCallback unsafe.Pointer var adBlockMatchers map[int32]*AdBlockMatcher +var adBlockInteropSyncMutex sync.Mutex + +var newAdBlockMatcher *AdBlockMatcher //export AdBlockMatcherInitialize func AdBlockMatcherInitialize() { - var oldMatcher *AdBlockMatcher = nil - - if adBlockMatcher != nil { - oldMatcher = adBlockMatcher - } + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() - adBlockMatcher = CreateMatcher() + newAdBlockMatcher = CreateMatcher() - if oldMatcher != nil { - adBlockMatcher.bypassEnabled = oldMatcher.bypassEnabled + if adBlockMatcher != nil { + newAdBlockMatcher.bypassEnabled = adBlockMatcher.bypassEnabled } } //export AdBlockMatcherBuild func AdBlockMatcherBuild() { - if adBlockMatcher != nil { - adBlockMatcher.Build() + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + + if newAdBlockMatcher != nil { + newAdBlockMatcher.Build() + + adBlockMatcher = newAdBlockMatcher + newAdBlockMatcher = nil } } //export AdBlockMatcherParseRuleFile func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listType int32) bool { - fileName := C.GoString(fileNameC) + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + categoryId := C.GoString(categoryIdC) + fileName := C.GoString(fileNameC) fileHandle, err := os.Open(fileName) if err != nil { @@ -50,38 +61,52 @@ func AdBlockMatcherParseRuleFile(fileNameC *C.char, categoryIdC *C.char, listTyp } defer fileHandle.Close() - scanner := bufio.NewScanner(fileHandle) + if newAdBlockMatcher == nil { + return false + } + scanner := bufio.NewScanner(fileHandle) log.Printf("Parsing category %s file %s", categoryId, fileName) - adBlockMatcher.addMatcher(categoryId, int(listType)) + newAdBlockMatcher.addMatcher(categoryId, int(listType)) if listType == TextTrigger { - adBlockMatcher.addPhrasesFromScanner(scanner, categoryId) + newAdBlockMatcher.addPhrasesFromScanner(scanner, categoryId) } else { - adBlockMatcher.addRulesFromScanner(scanner, categoryId, int(listType)) + newAdBlockMatcher.addRulesFromScanner(scanner, categoryId, int(listType)) } + time.Sleep(time.Millisecond * 10) return true } //export AdBlockMatcherSetBlockedPageContent func AdBlockMatcherSetBlockedPageContent(contentBlockPageC, contentCertPageC *C.char) { - blockPagePath := C.GoString(contentBlockPageC) - adBlockMatcher.BlockPageTemplate = parseTemplate(blockPagePath) + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() - certPagePath := C.GoString(contentCertPageC) - adBlockMatcher.BlockCertTemplate = parseTemplate(certPagePath) + if newAdBlockMatcher != nil { + blockPagePath := C.GoString(contentBlockPageC) + newAdBlockMatcher.BlockPageTemplate = parseTemplate(blockPagePath) + + certPagePath := C.GoString(contentCertPageC) + newAdBlockMatcher.BlockCertTemplate = parseTemplate(certPagePath) + } } //export AdBlockMatcherSetBlockPageContextTag func AdBlockMatcherSetBlockPageContextTag(keyC, valueC *C.char) { - key := C.GoString(keyC) - value := C.GoString(valueC) - - if len(value) > 0 { - adBlockMatcher.defaultBlockPageTags[key] = value - } else { - delete(adBlockMatcher.defaultBlockPageTags, key) + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + + if newAdBlockMatcher != nil { + key := C.GoString(keyC) + value := C.GoString(valueC) + + if len(value) > 0 { + newAdBlockMatcher.defaultBlockPageTags[key] = value + } else { + delete(newAdBlockMatcher.defaultBlockPageTags, key) + } } } @@ -109,16 +134,25 @@ func parseTemplate(pagePath string) *raymond.Template { //export AdBlockMatcherSave func AdBlockMatcherSave(fileName string) { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + adBlockMatcher.SaveToFile(fileName) } //export AdBlockMatcherLoad func AdBlockMatcherLoad(fileName string) { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + adBlockMatcher = LoadMatcherFromFile(fileName) } //export AdBlockMatcherEnableBypass func AdBlockMatcherEnableBypass() { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + if adBlockMatcher != nil { adBlockMatcher.bypassEnabled = true } @@ -126,6 +160,9 @@ func AdBlockMatcherEnableBypass() { //export AdBlockMatcherDisableBypass func AdBlockMatcherDisableBypass() { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + if adBlockMatcher != nil { adBlockMatcher.bypassEnabled = false } @@ -133,6 +170,9 @@ func AdBlockMatcherDisableBypass() { //export AdBlockMatcherGetBypassEnabled func AdBlockMatcherGetBypassEnabled() bool { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + if adBlockMatcher != nil { return adBlockMatcher.bypassEnabled } else { @@ -142,6 +182,9 @@ func AdBlockMatcherGetBypassEnabled() bool { //export AdBlockMatcherIsDomainWhitelisted func AdBlockMatcherIsDomainWhitelisted(hostC *C.char) bool { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + if adBlockMatcher != nil { host := C.GoString(hostC) return adBlockMatcher.IsDomainWhitelisted(host) @@ -152,6 +195,9 @@ func AdBlockMatcherIsDomainWhitelisted(hostC *C.char) bool { //export AdBlockMatcherGetWhitelistedDomains func AdBlockMatcherGetWhitelistedDomains() *C.char { + adBlockInteropSyncMutex.Lock() + defer adBlockInteropSyncMutex.Unlock() + if adBlockMatcher != nil { domains := adBlockMatcher.GetWhitelistedDomains() res := strings.Join(domains, ";") diff --git a/goproxy/certs.go b/goproxy/certs.go index b8a8e43..5b0ee48 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -20,25 +20,27 @@ import ( "github.com/cloudveiltech/goproxy" ) -var defaultTLSConfig = &tls.Config{ - Renegotiation: tls.RenegotiateFreelyAsClient, - InsecureSkipVerify: true, // We should be able to set this to false, and then check verified chains against peer certificates to see if we have a trusted chain or not. - VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - // for i := 0; i < len(rawCerts); i++ { - // cert, err := x509.ParseCertificate(rawCerts[i]) - - // if err != nil { - // fmt.Println("Error: ", err) - // continue - // } - - // hash := sha1.Sum(rawCerts[i]) - // fmt.Println("Cert data: ") - // fmt.Println(hash, cert.DNSNames, cert.Subject, cert.Issuer) - // } - - return nil - }, +func getDefaultTlsConfig() *tls.Config { + return &tls.Config{ + Renegotiation: tls.RenegotiateFreelyAsClient, + InsecureSkipVerify: true, // We should be able to set this to false, and then check verified chains against peer certificates to see if we have a trusted chain or not. + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + // for i := 0; i < len(rawCerts); i++ { + // cert, err := x509.ParseCertificate(rawCerts[i]) + + // if err != nil { + // fmt.Println("Error: ", err) + // continue + // } + + // hash := sha1.Sum(rawCerts[i]) + // fmt.Println("Cert data: ") + // fmt.Println(hash, cert.DNSNames, cert.Subject, cert.Issuer) + // } + + return nil + }, + } } var ( @@ -50,13 +52,11 @@ func loadAndSetCa(certFile, keyFile string) { cert, err := ioutil.ReadFile(certFile) if err != nil { log.Printf("Can't read cert file") - log.Fatal(err) return } key, err := ioutil.ReadFile(keyFile) if err != nil { log.Printf("Can't read cert key file") - log.Fatal(err) return } @@ -69,12 +69,10 @@ func setCA(caCert, caKey []byte) error { goproxyCa, err := tls.X509KeyPair(caCert, caKey) if err != nil { log.Printf("Can't load cert/key file") - log.Fatal(err) return err } if goproxyCa.Leaf, err = x509.ParseCertificate(goproxyCa.Certificate[0]); err != nil { log.Printf("Can't parse cert key/file") - log.Fatal(err) return err } goproxy.GoproxyCa = goproxyCa @@ -82,6 +80,8 @@ func setCA(caCert, caKey []byte) error { goproxy.MitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} goproxy.HTTPMitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectHTTPMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} goproxy.RejectConnect = &goproxy.ConnectAction{Action: goproxy.ConnectReject, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} + rootCert = goproxyCa + return nil } diff --git a/goproxy/debug.windows.go b/goproxy/debug.windows.go index 61ccc17..06da67c 100644 --- a/goproxy/debug.windows.go +++ b/goproxy/debug.windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package main @@ -28,7 +29,7 @@ func setStdHandle(stdhandle int32, handle syscall.Handle) error { func redirectStderr(f *os.File) { err := setStdHandle(syscall.STD_OUTPUT_HANDLE, syscall.Handle(f.Fd())) if err != nil { - log.Fatalf("Failed to redirect stderr to file: %v", err) + log.Printf("Failed to redirect stderr to file: %v", err) } // SetStdHandle does not affect prior references to stderr os.Stderr = f diff --git a/goproxy/goproxy_chain.go b/goproxy/goproxy_chain.go index 4471e0e..e6b6e4b 100644 --- a/goproxy/goproxy_chain.go +++ b/goproxy/goproxy_chain.go @@ -26,19 +26,21 @@ import ( ) func runConfigurationServerListener() { - go func() { - - cert, _ := signHost(goproxy.GoproxyCa, []string{"127.0.0.1"}) - config := defaultTLSConfig - config.Certificates = append(config.Certificates, *cert) - config.NextProtos = []string{"http/1.1"} - - srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", configuredConfigurationServerPort)} - srv.Handler = serverHandler{} - srv.TLSConfig = config - - srv.ListenAndServeTLS("", "") - }() + log.Printf("Starting configuration server %d", configuredConfigurationServerPort) + cert, _ := signHost(rootCert, []string{"127.0.0.1", "localhost"}) + config := getDefaultTlsConfig() + config.Certificates = append(config.Certificates, *cert) + config.NextProtos = []string{"http/1.1"} + + configServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", configuredConfigurationServerPort)} + configServer.Handler = serverHandler{} + configServer.TLSConfig = config + + if err := configServer.ListenAndServeTLS("", ""); err != nil { + // cannot panic, because this probably is an intentional close + log.Printf("configServer: ListenAndServe() error: %s", err) + configServer = nil + } } type serverHandler struct { @@ -99,10 +101,11 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { return } - start := time.Unix(0, 0) + end, err := time.Parse("2006-01-02", "2049-12-31") if err != nil { - panic(err) + log.Print("Can't parse date") + return } serial := big.NewInt(rand.Int63()) @@ -113,7 +116,7 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er Subject: pkix.Name{ Organization: []string{"GoProxy untrusted MITM proxy Inc"}, }, - NotBefore: start, + NotBefore: time.Unix(0, 0), NotAfter: end, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 18993c1..6ee591f 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -133,7 +133,7 @@ func runHttpsListener() { ln, err := net.Listen("tcp", fmt.Sprintf(":%d", configuredPortHttps)) if err != nil { - log.Fatalf("Error listening for https connections - %v", err) + log.Printf("Error listening for https connections - %v", err) return } @@ -263,6 +263,10 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert log.Printf("Page %s blocked by url, category %s", url, *category) + if strings.Contains(r.URL.Host, "vimeo") { + r.Header.Set("cookie", CookiePatchSafeSearch(r.URL.Host, r.Header.Get("cookie"))) + } + r.URL.RawPath = HostPathForceSafeSearch(r.URL.Host, r.URL.RawPath) return r, goproxy.NewResponse(r, goproxy.ContentTypeHtml, http.StatusForbidden, @@ -347,7 +351,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert return resp }) - runConfigurationServerListener() + go runConfigurationServerListener() go runHttpsListener() if proxy.Verbose { @@ -357,8 +361,14 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert func stopGoProxyServer() { if server != nil { - context, _ := context.WithTimeout(context.Background(), 1*time.Second) + context, _ := context.WithTimeout(context.Background(), 1*time.Millisecond) server.Shutdown(context) server = nil } + + if configServer != nil { + context, _ := context.WithTimeout(context.Background(), 1*time.Millisecond) + configServer.Shutdown(context) + configServer = nil + } } diff --git a/goproxy/http2.go b/goproxy/http2.go index fb84fef..b2a60b1 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -10,6 +10,7 @@ import ( "net/url" "strconv" "strings" + "sync" "sync/atomic" "github.com/cloudveiltech/goproxy" @@ -32,6 +33,7 @@ type Http2Handler struct { responseBodyMapChunks map[uint32][][]byte debouncers map[uint32]func(f func()) connectionReadyForData bool + rwMutex *sync.RWMutex } func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Conn) bool { @@ -47,6 +49,7 @@ func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Co responseBodyMapChunks: make(map[uint32][][]byte), debouncers: make(map[uint32]func(f func())), connectionReadyForData: false, + rwMutex: &sync.RWMutex{}, } go func() { http2Handler.processHttp2Stream(rawClientTls, remote) @@ -95,16 +98,12 @@ func isContentTypeFilterable(contentType string) bool { if strings.Contains(contentType, "protobuf") { return false } - - filterable := strings.Contains(contentType, "html") || strings.Contains(contentType, "json") - - if isImageFilteringEnabled && !filterable { - filterable = strings.Contains(contentType, "image/png") || - strings.Contains(contentType, "image/jpg") || - strings.Contains(contentType, "image/jpeg") || - strings.Contains(contentType, "image/webp") - } - return filterable + return strings.Contains(contentType, "html") || + strings.Contains(contentType, "json") || + strings.Contains(contentType, "image/png") || + strings.Contains(contentType, "image/jpg") || + strings.Contains(contentType, "image/jpeg") || + strings.Contains(contentType, "image/webp") } func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { @@ -120,14 +119,20 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F body := fr.Data() streamId := f.Header().StreamID + + http2Handler.rwMutex.RLock() lastHttpResponse := http2Handler.lastHttpResponse[streamId] bodyChunks := http2Handler.responseBodyMapChunks[streamId] + ctx := http2Handler.proxyCtx[streamId] + http2Handler.rwMutex.RUnlock() + chunk := make([]byte, len(body)) copy(chunk, body) bodyChunks = append(bodyChunks, chunk) - http2Handler.responseBodyMapChunks[streamId] = bodyChunks - ctx := http2Handler.proxyCtx[streamId] + http2Handler.rwMutex.Lock() + http2Handler.responseBodyMapChunks[streamId] = bodyChunks + http2Handler.rwMutex.Unlock() whitelisted := false if ctx != nil && ctx.UserData != nil { @@ -139,12 +144,10 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if force { log.Print("Force ending stream on timeout") } + http2Handler.rwMutex.RLock() lastHttpResponse = http2Handler.lastHttpResponse[streamId] bodyChunks = http2Handler.responseBodyMapChunks[streamId] - - if lastHttpResponse != nil && lastHttpResponse.Request != nil { - log.Printf("Process frame data %s", lastHttpResponse.Request.RequestURI) - } + http2Handler.rwMutex.RUnlock() streamEnded := fr.StreamEnded() || force if !whitelisted && lastHttpResponse != nil && !client { @@ -157,7 +160,10 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F isContentTypeFilterable = isContentTypeFilterable && (contentLength < MAX_FILTERABLE_LENGTH || isImage) if isContentTypeFilterable && streamEnded { if contentLength > MIN_FILTERABLE_LENGTH { + + http2Handler.rwMutex.RLock() ctx := http2Handler.proxyCtx[streamId] + http2Handler.rwMutex.RUnlock() resp := proxy.FilterResponse(lastHttpResponse, ctx) if resp != lastHttpResponse { //new response @@ -176,9 +182,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F buf.ReadFrom(resp.Body) writeFinalData(directFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) // directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) + http2Handler.rwMutex.Lock() delete(http2Handler.lastHttpResponse, streamId) delete(http2Handler.lastHttpRequest, streamId) delete(http2Handler.responseBodyMapChunks, streamId) + http2Handler.rwMutex.Unlock() return false } } @@ -187,14 +195,19 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F } } + http2Handler.rwMutex.RLock() header, ok := http2Handler.lastHeadersBlock[streamId] + http2Handler.rwMutex.RUnlock() if ok { // headerFields, _ := http2Handler.lastHeadersMap[streamId] header.EndStream = false // header.BlockFragment = encodeHeaderFields(headerFields) writeHeaders(directFramer, header, decoder) + + http2Handler.rwMutex.Lock() delete(http2Handler.lastHeadersBlock, streamId) delete(http2Handler.lastHeadersMap, streamId) + http2Handler.rwMutex.Unlock() } for i, _ := range bodyChunks { @@ -202,7 +215,9 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F directFramer.WriteData(streamId, streamEnd, bodyChunks[i]) } + http2Handler.rwMutex.Lock() delete(http2Handler.responseBodyMapChunks, streamId) + http2Handler.rwMutex.Unlock() return true } @@ -230,8 +245,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if client { request := makeHttpRequest(nil, headerFields) var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1)} + + http2Handler.rwMutex.Lock() http2Handler.lastHttpRequest[f.Header().StreamID] = request http2Handler.proxyCtx[f.Header().StreamID] = ctx + http2Handler.rwMutex.Unlock() _, resp := proxy.FilterRequest(request, ctx) if resp != nil { @@ -255,13 +273,17 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F } } else { response := makeHttpResponse(nil, headerFields) + + http2Handler.rwMutex.Lock() http2Handler.lastHttpResponse[f.Header().StreamID] = response + http2Handler.lastHttpResponse[f.Header().StreamID].Request = http2Handler.lastHttpRequest[f.Header().StreamID] + http2Handler.rwMutex.Unlock() + contentType := response.Header.Get("Content-Type") // contentLength, _ := strconv.Atoi(response.Header.Get("Content-Length")) if !isContentTypeFilterable(contentType) { writeHeadersImmediately = true } - http2Handler.lastHttpResponse[f.Header().StreamID].Request = http2Handler.lastHttpRequest[f.Header().StreamID] } header := http2.HeadersFrameParam{ @@ -276,8 +298,10 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if writeHeadersImmediately { writeHeaders(directFramer, &header, decoder) } else { + http2Handler.rwMutex.Lock() http2Handler.lastHeadersMap[f.Header().StreamID] = headerFields http2Handler.lastHeadersBlock[f.Header().StreamID] = &header + http2Handler.rwMutex.Unlock() } case http2.FramePriority: fr := f.(*http2.PriorityFrame) @@ -347,6 +371,7 @@ func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpa hostIndex := 0 pathIndex := 0 + cookieIndex := 0 decoder.SetEmitEnabled(true) decoder.SetMaxStringLength(16 << 20) decoder.SetEmitFunc(func(hf hpack.HeaderField) { @@ -355,6 +380,8 @@ func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpa pathIndex = len(res) } else if hf.Name == ":authority" { hostIndex = len(res) + } else if hf.Name == "cookie" { + cookieIndex = len(res) } res = append(res, hf) } @@ -370,6 +397,10 @@ func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpa if fr.HeadersEnded() { if hostIndex > 0 || pathIndex > 0 { res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + + if cookieIndex > 0 { + res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) + } } return res, buf.Bytes() @@ -392,6 +423,9 @@ func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpa if hostIndex > 0 || pathIndex > 0 { res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + if cookieIndex > 0 { + res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) + } } return res, buf.Bytes() diff --git a/goproxy/main.go b/goproxy/main.go index 39edcf2..0aa872f 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -10,6 +10,7 @@ import ( "net" "os" "runtime/debug" + "sync" "unsafe" ) @@ -26,8 +27,13 @@ var logFilePath = "" var logFileHandle *os.File var isImageFilteringEnabled = false +var mainInteropSyncMutex sync.Mutex + //export AddCertException func AddCertException(thumbPrintC *C.char) { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + thumbPrint := C.GoString(thumbPrintC) _, ok := certsException[thumbPrint] if !ok { @@ -53,6 +59,9 @@ func checkPortAvailable(port int16) bool { //export SetProxyLogFile func SetProxyLogFile(logFile *C.char) { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + logFilePath = C.GoString(logFile) setProxyLogFileInternal(logFilePath) } @@ -105,11 +114,17 @@ func monitorLogFileSize() { //export AdBlockMatcherSetBlacklistCallback func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + adBlockBlacklistCallback = callback } //export StartGoServer func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char, bannedImageFileC *C.char) int16 { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + debug.SetTraceback("all") debug.SetPanicOnFault(true) initIpUtil() @@ -143,17 +158,26 @@ func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC //export SetImageFilteringEnabled func SetImageFilteringEnabled(enabled bool) { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + log.Printf("Setting Image filtering to: %v", enabled) isImageFilteringEnabled = enabled } //export StopGoServer func StopGoServer() { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + stopGoProxyServer() } //export IsIpPrivate func IsIpPrivate(ipStringC *C.char) int16 { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + ipString := C.GoString(ipStringC) ip := net.ParseIP(ipString) if ip == nil { diff --git a/goproxy/request.go b/goproxy/request.go index 03feeac..4fc0ee8 100644 --- a/goproxy/request.go +++ b/goproxy/request.go @@ -2,132 +2,11 @@ package main import ( "C" - "bytes" ) import ( - "io/ioutil" "strings" ) -//export RequestGetUrl -func RequestGetUrl(id int64, result *string) bool { - request := getSessionRequest(id) - - if request == nil { - return false - } - - if request.URL == nil { - return false - } - - *result = request.URL.String() - return len(*result) > 0 -} - -//export RequestGetBody -func RequestGetBody(id int64, res *[]byte) bool { - request := getSessionRequest(id) - if request == nil { - return false - } - - if request.Body == nil { - return false - } - - buf := new(bytes.Buffer) - buf.ReadFrom(request.Body) - - *res = buf.Bytes() - - //since we'd read all body - we need to recreate reader for client here - request.Body.Close() - request.Body = ioutil.NopCloser(bytes.NewBuffer(*res)) - - return true -} - -//export RequestGetBodyAsString -func RequestGetBodyAsString(id int64, res *string) bool { - var bytes []byte - if !RequestGetBody(id, &bytes) { - return false - } - *res = string(bytes[:]) - - return true -} - -//export RequestHasBody -func RequestHasBody(id int64) bool { - request := getSessionRequest(id) - if request == nil { - return false - } - - return request.Body != nil && request.ContentLength != 0 -} - -//export RequestHeaderExists -func RequestHeaderExists(id int64, name string) bool { - request := getSessionRequest(id) - if request == nil { - return false - } - - // for k := range request.Header { - // fmt.Fprintf(os.Stderr, "key[%s] value[%s]\n", k, request.Header[k]) - // } - - _, headerExists := request.Header[name] - return headerExists -} - -//export RequestGetFirstHeader -func RequestGetFirstHeader(id int64, name string, res *string) bool { - request := getSessionRequest(id) - if request == nil { - return false - } - - values, headerExists := request.Header[name] - if !headerExists { - return false - } - *res = values[0] - return true -} - -//export RequestSetHeader -func RequestSetHeader(id int64, name string, value string) bool { - request := getSessionRequest(id) - if request == nil { - return false - } - - request.Header.Set(name, value) - return true -} - -//export RequestGetHeaders -func RequestGetHeaders(id int64, keys *string) int { - request := getSessionRequest(id) - if request == nil { - return 0 - } - var result strings.Builder - for key, v := range request.Header { - for _, value := range v { - result.WriteString(key + ": " + value + "\r\n") - } - } - - *keys = result.String() - - return len(request.Header) -} - func HostPathForceSafeSearch(host, path string) string { // enforce Google safe-search if strings.Contains(host, "google.com") && strings.Contains(path, "/search?") && !strings.Contains(path, "safe=active") { @@ -141,3 +20,20 @@ func HostPathForceSafeSearch(host, path string) string { } return path } + +func CookiePatchSafeSearch(host, cookieValue string) string { + if strings.Contains(host, "vimeo") { + cookieParts := strings.Split(cookieValue, ";") + newCookieParts := make([]string, 0) + for _, cookie := range cookieParts { + if !strings.Contains(cookie, "content_rating") { + newCookieParts = append(newCookieParts, cookie) + } + } + + newCookieParts = append(newCookieParts, "content_rating=7") + + return strings.Join(newCookieParts, ";") + } + return cookieValue +} diff --git a/goproxy/response.go b/goproxy/response.go index 3922272..323e55e 100644 --- a/goproxy/response.go +++ b/goproxy/response.go @@ -7,68 +7,13 @@ import ( "bytes" "io" "io/ioutil" - "log" - "strings" "compress/flate" "compress/gzip" - "github.com/cloudveiltech/goproxy" "github.com/dsnet/compress/brotli" ) -//export ResponseGetStatusCode -func ResponseGetStatusCode(id int64) int { - response := getSessionResponse(id) - if response == nil { - return 0 - } - - return response.StatusCode -} - -//export ResponseGetBody -func ResponseGetBody(id int64, res *[]byte) bool { - response := getSessionResponse(id) - if response == nil { - return false - } - - if response.Body == nil { - return false - } - - buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) - - result := buf.Bytes() - if response.Uncompressed { - *res = result - } else { - *res = decodeResponseCompression(response.Header.Get("Content-Encoding"), result) - if *res == nil { - *res = result - } - } - - //since we'd read all body - we need to recreate reader for client here - response.Body.Close() - response.Body = ioutil.NopCloser(bytes.NewBuffer(result)) - - return true -} - -//export ResponseGetBodyAsString -func ResponseGetBodyAsString(id int64, res *string) bool { - var bytes []byte - if !ResponseGetBody(id, &bytes) { - return false - } - *res = string(bytes[:]) - - return true -} - func decodeResponseCompression(contentEncoding string, body []byte) []byte { switch contentEncoding { case "gzip": @@ -102,114 +47,3 @@ func readReader(reader io.ReadCloser, err error) []byte { } return nil } - -//export ResponseHasBody -func ResponseHasBody(id int64) bool { - response := getSessionResponse(id) - if response == nil { - return false - } - - return response.Body != nil && response.ContentLength != 0 -} - -//export ResponseHeaderExists -func ResponseHeaderExists(id int64, name string) bool { - response := getSessionResponse(id) - if response == nil { - return false - } - - _, headerExists := response.Header[name] - return headerExists -} - -//export ResponseGetFirstHeader -func ResponseGetFirstHeader(id int64, name string, res *string) bool { - response := getSessionResponse(id) - if response == nil { - return false - } - - values, headerExists := response.Header[name] - if !headerExists { - return false - } - *res = values[0] - return true -} - -//export ResponseSetHeader -func ResponseSetHeader(id int64, name string, value string) bool { - response := getSessionResponse(id) - if response == nil { - return false - } - - response.Header.Set(name, value) - return true -} - -//export ResponseGetHeaders -func ResponseGetHeaders(id int64, keys *string) int { - response := getSessionResponse(id) - if response == nil { - return 0 - } - var result strings.Builder - for key, v := range response.Header { - for _, value := range v { - result.WriteString(key + ": " + value + "\r\n") - } - } - - *keys = result.String() - return len(response.Header) -} - -//export ResponseGetCertificatesCount -func ResponseGetCertificatesCount(id int64) int { - response := getSessionResponse(id) - if response == nil { - return 0 - } - if response.TLS == nil { - return 0 - } - return len(response.TLS.PeerCertificates) -} - -//export ResponseIsTLSVerified -func ResponseIsTLSVerified(id int64) bool { - isVerified := isSessionTlsVerified(id) - return isVerified -} - -//export ResponseGetCertificate -func ResponseGetCertificate(id int64, index int32, certData *[]byte) int { - response := getSessionResponse(id) - if response == nil { - return 0 - } - if response.TLS == nil { - return 0 - } - - cert := response.TLS.PeerCertificates[index] - - *certData = cert.Raw - - return 1 -} - -//export CreateResponse -func CreateResponse(id int64, status int32, contentType string, body string) bool { - session, exists := sessionMap[id] - if !exists { - log.Printf("NO SESSION FOR RESPONSE %d", id) - return false - } - - session.response = goproxy.NewResponse(session.request, contentType, int(status), body) - return true -} From e2875c2d935a3ce483aa6cd2121834cfaf45ba41 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Tue, 13 Jun 2023 13:30:31 +0200 Subject: [PATCH 22/27] merged win branch --- goproxy/certs.go | 14 ++++++-- goproxy/goproxy_chain.go | 5 +-- goproxy/goproxystarter.go | 30 +++++++++++++----- goproxy/http2.go | 67 ++++++++++++++++++++++++--------------- 4 files changed, 78 insertions(+), 38 deletions(-) diff --git a/goproxy/certs.go b/goproxy/certs.go index 5b0ee48..d7e7176 100644 --- a/goproxy/certs.go +++ b/goproxy/certs.go @@ -5,7 +5,6 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/rsa" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -17,7 +16,10 @@ import ( "strings" "time" + oldTls "crypto/tls" + "github.com/cloudveiltech/goproxy" + tls "github.com/refraction-networking/utls" ) func getDefaultTlsConfig() *tls.Config { @@ -80,8 +82,16 @@ func setCA(caCert, caKey []byte) error { goproxy.MitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} goproxy.HTTPMitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectHTTPMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} goproxy.RejectConnect = &goproxy.ConnectAction{Action: goproxy.ConnectReject, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} - rootCert = goproxyCa + rootCert, err = oldTls.X509KeyPair(caCert, caKey) + if err != nil { + log.Printf("Can't load cert/key file") + return err + } + if rootCert.Leaf, err = x509.ParseCertificate(rootCert.Certificate[0]); err != nil { + log.Printf("Can't parse cert key/file") + return err + } return nil } diff --git a/goproxy/goproxy_chain.go b/goproxy/goproxy_chain.go index e6b6e4b..1fd317d 100644 --- a/goproxy/goproxy_chain.go +++ b/goproxy/goproxy_chain.go @@ -7,7 +7,6 @@ import ( "crypto/elliptic" "crypto/rsa" "crypto/sha1" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "fmt" @@ -22,13 +21,15 @@ import ( "strings" "time" + "crypto/tls" + "github.com/cloudveiltech/goproxy" ) func runConfigurationServerListener() { log.Printf("Starting configuration server %d", configuredConfigurationServerPort) cert, _ := signHost(rootCert, []string{"127.0.0.1", "localhost"}) - config := getDefaultTlsConfig() + config := &tls.Config{} config.Certificates = append(config.Certificates, *cert) config.NextProtos = []string{"http/1.1"} diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 6ee591f..ccb7ae7 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -17,7 +17,6 @@ import ( "bufio" "bytes" "context" - "crypto/tls" "encoding/base64" "encoding/binary" "fmt" @@ -86,13 +85,6 @@ func initGoProxy() { MaxIdleConns: 1000, IdleConnTimeout: time.Minute * 10, ResponseHeaderTimeout: time.Minute * 10, - TLSClientConfig: &tls.Config{ - NextProtos: []string{"http/1.1"}, - InsecureSkipVerify: true, - CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256}, - PreferServerCipherSuites: true, - Renegotiation: tls.RenegotiateFreelyAsClient, - }, } proxy.OnRequest().HandleConnect(handleConnectFunc) @@ -167,6 +159,12 @@ func runHttpsListener() { return } + if adBlockMatcher.IsDomainWhitelisted(host) { + log.Printf("Early whitelisting https host %s", host) + chainReqWithoutFiltering(tlsConn, host, port) + return + } + host = net.JoinHostPort(host, strconv.Itoa(int(port))) resp := dumbResponseWriter{tlsConn} connectReq := &http.Request{ @@ -184,6 +182,22 @@ func runHttpsListener() { } } +func chainReqWithoutFiltering(client net.Conn, host string, port uint16) { + defer client.Close() + remote, err := net.Dial("tcp", net.JoinHostPort(host, strconv.Itoa(int(port)))) + if err != nil { + log.Printf("chainReqWithoutFiltering error connect %s", err) + return + } + + defer remote.Close() + + go func() { + io.Copy(remote, client) + }() + + io.Copy(client, remote) +} func chainReqToHttp(client net.Conn) { chainReqToHost(client, fmt.Sprintf("127.0.0.1:%d", configuredPortHttp)) } diff --git a/goproxy/http2.go b/goproxy/http2.go index b2a60b1..fe48851 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "crypto/tls" "io" "io/ioutil" "log" @@ -16,6 +15,8 @@ import ( "github.com/cloudveiltech/goproxy" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + + tls "github.com/refraction-networking/utls" ) var http2ProxySessionCounter int64 @@ -23,6 +24,10 @@ var http2ProxySessionCounter int64 const MAX_FILTERABLE_LENGTH = 1024 * 1024 const MIN_FILTERABLE_LENGTH = 100 +const STATUS_BLOCKED = 0 +const STATUS_OK = 1 +const STATUS_ENDED = 2 + type Http2Handler struct { maxFrameSize uint32 lastHttpResponse map[uint32]*http.Response @@ -36,7 +41,7 @@ type Http2Handler struct { rwMutex *sync.RWMutex } -func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Conn) bool { +func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UConn) bool { log.Print("Running http2 handler for " + r.URL.String()) http2Handler := &Http2Handler{ @@ -58,7 +63,7 @@ func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.Co return true } -func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tls.Conn) { +func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tls.UConn) { const preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" b := make([]byte, len(preface)) if _, err := io.ReadFull(local, b); err != nil { @@ -76,19 +81,21 @@ func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tl reverseFramer := http2.NewFramer(local, remote) go func() { - defer remote.Close() - defer local.Close() decoder := hpack.NewDecoder(65536, nil) for { - if !http2Handler.readFrame(reverseFramer, directFramer, decoder, false) { + res := http2Handler.readFrame(reverseFramer, directFramer, decoder, false) + if res != STATUS_OK { return } } }() - decoder := hpack.NewDecoder(65536, nil) for { - if !http2Handler.readFrame(directFramer, reverseFramer, decoder, true) { + res := http2Handler.readFrame(directFramer, reverseFramer, decoder, true) + if res != STATUS_OK { + if res == STATUS_BLOCKED { + remote.Close() + } return } } @@ -106,11 +113,11 @@ func isContentTypeFilterable(contentType string) bool { strings.Contains(contentType, "image/webp") } -func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { +func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) int { f, err := directFramer.ReadFrame() if err != nil { log.Printf("ReadFrame client %v, err: %v", client, err) - return false + return STATUS_ENDED } switch f.Header().Type { @@ -140,7 +147,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F whitelisted = exists && !(blocked.(bool)) } - processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) bool { + processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) int { if force { log.Print("Force ending stream on timeout") } @@ -160,7 +167,6 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F isContentTypeFilterable = isContentTypeFilterable && (contentLength < MAX_FILTERABLE_LENGTH || isImage) if isContentTypeFilterable && streamEnded { if contentLength > MIN_FILTERABLE_LENGTH { - http2Handler.rwMutex.RLock() ctx := http2Handler.proxyCtx[streamId] http2Handler.rwMutex.RUnlock() @@ -187,11 +193,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F delete(http2Handler.lastHttpRequest, streamId) delete(http2Handler.responseBodyMapChunks, streamId) http2Handler.rwMutex.Unlock() - return false + return STATUS_BLOCKED } } } else if isContentTypeFilterable { - return true + return STATUS_OK } } @@ -218,7 +224,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F http2Handler.rwMutex.Lock() delete(http2Handler.responseBodyMapChunks, streamId) http2Handler.rwMutex.Unlock() - return true + return STATUS_OK } processDataFrameFunc(false, streamId, directFramer, reverseFramer, decoder, client) @@ -241,14 +247,23 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F if len(headerFields) == 0 { log.Printf("Error parsing headers") } - writeHeadersImmediately := client || fr.StreamEnded() + whitelisted := false + + streamId := f.Header().StreamID + ctx := http2Handler.proxyCtx[streamId] + if ctx != nil && ctx.UserData != nil { + blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] + whitelisted = exists && !(blocked.(bool)) + } + + writeHeadersImmediately := whitelisted || client || fr.StreamEnded() if client { request := makeHttpRequest(nil, headerFields) var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1)} http2Handler.rwMutex.Lock() - http2Handler.lastHttpRequest[f.Header().StreamID] = request - http2Handler.proxyCtx[f.Header().StreamID] = ctx + http2Handler.lastHttpRequest[streamId] = request + http2Handler.proxyCtx[streamId] = ctx http2Handler.rwMutex.Unlock() _, resp := proxy.FilterRequest(request, ctx) @@ -257,7 +272,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F reverseFramer.WriteSettings() } writeHeaders(reverseFramer, &http2.HeadersFrameParam{ - StreamID: f.Header().StreamID, + StreamID: streamId, BlockFragment: encodeHeaders(resp), EndStream: false, EndHeaders: true, @@ -267,16 +282,16 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F buf := new(bytes.Buffer) buf.ReadFrom(resp.Body) - writeFinalData(reverseFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) - reverseFramer.WriteGoAway(f.Header().StreamID, http2.ErrCodeRefusedStream, nil) - return false + writeFinalData(reverseFramer, streamId, buf, int(http2Handler.maxFrameSize)) + reverseFramer.WriteGoAway(streamId, http2.ErrCodeRefusedStream, nil) + return STATUS_BLOCKED } } else { response := makeHttpResponse(nil, headerFields) http2Handler.rwMutex.Lock() - http2Handler.lastHttpResponse[f.Header().StreamID] = response - http2Handler.lastHttpResponse[f.Header().StreamID].Request = http2Handler.lastHttpRequest[f.Header().StreamID] + http2Handler.lastHttpResponse[streamId] = response + http2Handler.lastHttpResponse[streamId].Request = http2Handler.lastHttpRequest[streamId] http2Handler.rwMutex.Unlock() contentType := response.Header.Get("Content-Type") @@ -287,7 +302,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F } header := http2.HeadersFrameParam{ - StreamID: f.Header().StreamID, + StreamID: streamId, BlockFragment: encodeHeaderFields(headerFields), EndStream: fr.StreamEnded(), EndHeaders: fr.HeadersEnded(), @@ -362,7 +377,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F directFramer.WriteRawFrame(f.Header().Type, f.Header().Flags, f.Header().StreamID, fr.Payload()) } - return true + return STATUS_OK } func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpack.Decoder) ([]hpack.HeaderField, []byte) { From 32f905f222342bc39f74ca39ff45d6de0178d795 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Mon, 10 Jul 2023 16:46:30 +0700 Subject: [PATCH 23/27] ipv6 fixes merged --- goproxy/goproxystarter.go | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index ccb7ae7..746f48c 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -198,8 +198,50 @@ func chainReqWithoutFiltering(client net.Conn, host string, port uint16) { io.Copy(client, remote) } + func chainReqToHttp(client net.Conn) { - chainReqToHost(client, fmt.Sprintf("127.0.0.1:%d", configuredPortHttp)) + localAddress := client.LocalAddr().(*net.TCPAddr).IP + + remote, err := net.Dial("tcp", net.JoinHostPort(localAddress.String(), strconv.Itoa(int(configuredPortHttp)))) + if err != nil { + log.Printf("chainReqToHttp error connect %s", err) + return + } + + defer remote.Close() + defer client.Close() + + go func() { + nonBlockingCopy(remote, client) + }() + + nonBlockingCopy(client, remote) +} + +func nonBlockingCopy(from, to net.Conn) { + buf := make([]byte, 10240) + for { + from.SetDeadline(time.Now().Add(time.Minute * 10)) + if server == nil { + log.Printf("Break chain on server stop") + break + } + + n, err := from.Read(buf) + if err != nil && err != io.EOF { + log.Printf("error request %s", err) + break + } + if n == 0 { + break + } + + if _, err := to.Write(buf[:n]); err != nil { + log.Printf("error response %s", err) + break + } + + } } func chainReqToHost(client net.Conn, hostPort string) { @@ -221,7 +263,7 @@ func chainReqToHost(client net.Conn, hostPort string) { } func startHttpServer(port int16) *http.Server { - srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port)} + srv := &http.Server{Addr: fmt.Sprintf(":%d", port)} srv.Handler = proxy proxy.Http2Handler = serveHttp2Filtering From f4f9535b89ad7719a86ba566623efb8dfcbb7fe0 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Thu, 21 Nov 2024 14:51:04 +0100 Subject: [PATCH 24/27] add cert api fix ssl errors bump goproxy version --- goproxy/adblock.go | 4 ++ goproxy/cert.pem | 61 +++++++++++----------- goproxy/goproxystarter.go | 47 +++++++++++++---- goproxy/key.pem | 103 +++++++++++++++++++------------------- goproxy/main.go | 11 ++++ 5 files changed, 137 insertions(+), 89 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 2d6ea24..42ff99f 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -164,6 +164,10 @@ func (am *AdBlockMatcher) GetBadCertPage(blockedUrl, host, certThumbPrint string } func (am *AdBlockMatcher) IsDomainWhitelisted(host string) bool { + if strings.Contains(host, "[::1]") { + log.Printf("::1 whitelist!") + return true + } category, matchType, _ := am.TestUrlBlocked("https://"+host, host, "") if category != nil && matchType == Excluded { log.Printf("Testing early host - true %s", host) diff --git a/goproxy/cert.pem b/goproxy/cert.pem index f539c74..e4bc027 100644 --- a/goproxy/cert.pem +++ b/goproxy/cert.pem @@ -1,31 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIUVC7fYFzzKewOyuPUQKF0yA+oHrYwDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0xODExMDkxNjU4MDBaFw0yODEx -MDYxNjU4MDBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQCkYZ+WQeA/PNH5fE+MWvG6z70i66nCtdtYsklBzDTS -ODYKn1hE2qHNiDT4iC2BqBsca6pQUkGG+GfvJuMp7vFb0ensHRUJFo21alHb9c9p -JstHnBOG9Vf1lpIBTtgVX6eBk6Iqrxh5YvCpuHnK9O1xYzwM9PYtula+4nEe6lmf -GlEPcvWfLTvsBl5AbPoXEqahwAN7QeDbW1jjP6o4+KOwERFuTRNp3P3m7Wz8Cx6N -U+4jErifMX4M1XbuE0kjC5ITj8pTi1LvsWsSojI2lYtw83tq8iWcKWuoWlLACRKw -4yzHGblyvn38wteeF7x61A2N7GjEX0z1eTn5lK/7XaYd8+SphwoV7xRWn+7g7ODy -5E/2McAPXi/R5sC1y6XgjcefMO4MiLaCvzxgdZjP9P4UawOOwP0GgnZsMLnLk4qn -DrQLIZwtfmWOG9wnjdQZ6yKRHLBfdEM2G67764TxIwDY93/i0IYBZlAMfxxRCrvK -AQkd8epSInz3PVN0PeBGvISMsBvNvG3Gkldt2SNz9bnWdho1wDA31P945ZaUetvU -8jOrLemC2rTJ1Dit1qaBTOVekUf3ktdfMCzgP1nte51r1ody1LFKFr1XdsadD2jZ -Rb1LeW1+AQyAAYHIO+rnKJLSX3l3wi9xTfwUBQD+6LpntkQsPfzQ4Xqurby8WbjG -ZwIDAQABo1MwUTAdBgNVHQ4EFgQUQKjMUPSq9vvhHfjQAXXlp8UCKWEwHwYDVR0j -BBgwFoAUQKjMUPSq9vvhHfjQAXXlp8UCKWEwDwYDVR0TAQH/BAUwAwEB/zANBgkq -hkiG9w0BAQsFAAOCAgEAnVa5uXbq3lEkrEfmclf82GIvHcMGOksmJQNzAKBgI06l -NrKKHZwhNorZq407182eYNPP644FOLgulj1vIMs+4XmqgOQDY+b7yn4IwMs5EnFZ -vSeKLNBchzqIo3ZQzR0tLEW6lT7THbq8m4acQ8vMThNgD6+s/x6FacKAgwxcEHwp -Z76CtSTHBP5TMPPSFtPN+2dcyT2zGzfqZlWjuAnu4epA8bgJXzHbhoDvpQgNGQ0A -tUGfkqgZSL5qEMlFEB/B0x1yIn2E1+4yDlyfjjkGTpT4aJn5+ln2v+TMdTW6j7SO -VQP2siWBpg73ZcbBys1xlKatgjyMP2T590m8Tu459/FxT/zUSkL09ywB/kYxPawB -XLrypYHG9KhqL5NBKEyPv5+MdgQ9PjrpUcTjs/e9iwyfjiZ7nR42cRlXokhtbOme -LFntHzNtTx3ROP1xI65EzqOExhghEJABEtY16QFklBRSswg1UhU7mfZzXc1HQoGs -9hQ4IcItYmd8tU1DJYPx3bw4Lvsps0dSMDc/ixE0wzuOxMa2dXExzauhKcKc7fmR -FFmJWFOhUbN5OuoZXYj57uXaZnOm4Hp8G9XpCraAYi79AiAP1ybnnu3oJOuoTceg -zYINomf+SAYAZuyoRdwRfk6m7ZGAcAySLCS1xl18bSP8e069EfzPc0t5ar4eIXo= +MIIF7zCCA9egAwIBAgIUEWeg/SNSBqHsYt1SWN+Nnxcd3IowDQYJKoZIhvcNAQEL +BQAwgYYxCzAJBgNVBAYTAlhYMRIwEAYDVQQIDAlTdGF0ZU5hbWUxETAPBgNVBAcM +CENpdHlOYW1lMRQwEgYDVQQKDAtDb21wYW55TmFtZTEbMBkGA1UECwwSQ29tcGFu +eVNlY3Rpb25OYW1lMR0wGwYDVQQDDBRDb21tb25OYW1lT3JIb3N0bmFtZTAeFw0y +NDA5MTExMTE2NThaFw0zNDA5MDkxMTE2NThaMIGGMQswCQYDVQQGEwJYWDESMBAG +A1UECAwJU3RhdGVOYW1lMREwDwYDVQQHDAhDaXR5TmFtZTEUMBIGA1UECgwLQ29t +cGFueU5hbWUxGzAZBgNVBAsMEkNvbXBhbnlTZWN0aW9uTmFtZTEdMBsGA1UEAwwU +Q29tbW9uTmFtZU9ySG9zdG5hbWUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQDbyevJD/4RdUvOnQnKdWc7gSocTWHXrOwZowunekDknOlU9Ly5BHsQixna +a4jtQ5zjtHmgZvfFQDQF5S0WYqPgNxHlUC0iYl+oVtDolKaZkNN1U8I1CGAh1gAV +ZcG4yqx0XWh4+tq1kj38RfEeSt4dkGXUmsxwNeh309CUpuJHT5SnD7oLDMnd/mjf +ZGTjR6//uaamXSiGFSlcIkH1ED+peKgcAGqC6KW+ckHBvCd4Pwzv7mPCDFkOARqi +PkUEGidk2J5hxHN35EYjNechYJ1SQKWQFgTpyJkjMG+YG6Z/SFg6PCZCBIrW2sou +AIQU1l9FlAOvn0A4Z4TUpZ1Z3/6hJR0+b0yi8nuy00ZfFQjSWSBiitobWZwIveH+ +ezG6xWMGtYkHxT71AO1R9m4lnwXYNwBMNi2F5MRT+7AIeE65RD0P0hZFBL9AQz80 +F942EgK/djETpTWuUoEanAm9SiCfOjCj1OsDpdEyFrBohCOAY99/LB+LohiXpbU8 +wb6S20qOdhgvJT+UumLIKRXE00KwL5n/e/RpyaGDJnd87XgkST28lj6hsqF+ToCX +K1vN0zaF08d5cUb0lgz8As3IrtuWCN6nyHj7rwZJltQjgWh8vgF5ecY4FAtr0DKY +9vP2NjVBDhDDAnFv+rDTX/1Esz6NzmnpT/jGWFpcKSkHybzvOwIDAQABo1MwUTAd +BgNVHQ4EFgQU9R656DTQ9phkZd7Prb8OcJUnbrIwHwYDVR0jBBgwFoAU9R656DTQ +9phkZd7Prb8OcJUnbrIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC +AgEAcjcgEllxnSJQ9s4Kz1MAkGr2GLmHnK2k/hH1mU4wRu8YCwzDm3T3/upSRmms +1jHG72571UEyejfFNQ/BYO6uJ3TjrQSzRYPV5YOnI5HsJVu0NpbR2KE/jQYFc0Vl +aw9UNjSSwfOtjsaV1Q5OsYtRqDOrClQ7zuiAMJUNkmul6U7urhMb+vy9HhAL6G1j +BN1Cjdk3ZGXRJPxQWEyl6ufOwGVbmI3si2gcLMjGMaJlg1d7FVz4on98b2ykETK6 +V7JJ7RhLeT7p2eWhFE60Ooozb4NVuOqrjSH9ZlTiU537nLKh+abZdgGHnt/Q0pKD +JaQOFZAFiwSQl3LY9kEz7t0DHmVF/Y+SZrEhrF50qus1r+VPL+BAWmwO6yz6bIAL +wrFjdStfLF44OgZjz4JpY3kyrdzi2lbtd9DKeiOzqs7GpOSME8mjm00tCd2dpUPa +SHOraFzmFECPzjiXZ2F7QuQOjUZ+qGJF7ldUADpG15kc6BIn2NJZ0aqIa9wtaH7x +QKquGCarRdBFYJwkAmZQAs44OwXDlaMZmyoxiUgIWcK6Ge4oItBtDHT6pFdZDn+U +fmRMxksiHo4OaoxdDypJXV+Ef1+3hEreUYIAQPwqMCdRkSjvwjzB8ARgVM5efWK+ +wQOkW3gyIUvC+rqrJ9WaXhljnA1Hmy+rowjpP/phepqQXyQ= -----END CERTIFICATE----- diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 746f48c..5c59296 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -143,20 +143,38 @@ func runHttpsListener() { port := DEFAULT_HTTPS_PORT if n > 0 { port = binary.BigEndian.Uint16([]byte{helloBuffer[1], helloBuffer[0]}) - log.Printf("Reading dest port for %d", port) + + add := c.RemoteAddr().String() + log.Printf("Reading dest port for %s :%d", add, port) } tlsConn, err := vhost.TLS(c) if err != nil { - log.Printf("Assuming plain http connection - %v", err) - chainReqToHttp(tlsConn) - return + log.Printf("Error reading TLS connection - %v", err) + if port != DEFAULT_HTTPS_PORT { + chainReqToHttp(tlsConn) + return + } } host := tlsConn.Host() if host == "" { - log.Printf("Cannot support client") - return + host = tlsConn.LocalAddr().String() + remoteHost := tlsConn.RemoteAddr().String() + log.Printf("Cannot support client trying host %s remote: %s, https port %d, http port %d", host, remoteHost, configuredPortHttps, configuredPortHttp) + + parsedHost, _, err := net.SplitHostPort(host) + if err == nil { + ip := net.ParseIP(parsedHost) + isPrivateNetwork := ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalMulticast() + if isPrivateNetwork { + if proxy.Verbose { + log.Printf("Chain local IP wihout filtering %s", host) + } + chainReqWithoutFilteringToAddress(tlsConn, host) + return + } + } } if adBlockMatcher.IsDomainWhitelisted(host) { @@ -184,10 +202,21 @@ func runHttpsListener() { func chainReqWithoutFiltering(client net.Conn, host string, port uint16) { defer client.Close() - remote, err := net.Dial("tcp", net.JoinHostPort(host, strconv.Itoa(int(port)))) + address := net.JoinHostPort(host, strconv.Itoa(int(port))) + chainReqWithoutFilteringToAddress(client, address) +} + +func chainReqWithoutFilteringToAddress(client net.Conn, address string) { + defer client.Close() + log.Printf("Chain without filtering %s", address) + dialer := net.Dialer{Timeout: time.Minute} + remote, err := dialer.Dial("tcp", address) if err != nil { - log.Printf("chainReqWithoutFiltering error connect %s", err) - return + remote, err = dialer.Dial("tcp", address) + if err != nil { + log.Printf("chainReqWithoutFiltering error connect %s", err) + return + } } defer remote.Close() diff --git a/goproxy/key.pem b/goproxy/key.pem index 2d1c3d7..851cb98 100644 --- a/goproxy/key.pem +++ b/goproxy/key.pem @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJJQIBAAKCAgEApGGflkHgPzzR+XxPjFrxus+9IuupwrXbWLJJQcw00jg2Cp9Y -RNqhzYg0+IgtgagbHGuqUFJBhvhn7ybjKe7xW9Hp7B0VCRaNtWpR2/XPaSbLR5wT -hvVX9ZaSAU7YFV+ngZOiKq8YeWLwqbh5yvTtcWM8DPT2LbpWvuJxHupZnxpRD3L1 -ny077AZeQGz6FxKmocADe0Hg21tY4z+qOPijsBERbk0Tadz95u1s/AsejVPuIxK4 -nzF+DNV27hNJIwuSE4/KU4tS77FrEqIyNpWLcPN7avIlnClrqFpSwAkSsOMsxxm5 -cr59/MLXnhe8etQNjexoxF9M9Xk5+ZSv+12mHfPkqYcKFe8UVp/u4Ozg8uRP9jHA -D14v0ebAtcul4I3HnzDuDIi2gr88YHWYz/T+FGsDjsD9BoJ2bDC5y5OKpw60CyGc -LX5ljhvcJ43UGesikRywX3RDNhuu++uE8SMA2Pd/4tCGAWZQDH8cUQq7ygEJHfHq -UiJ89z1TdD3gRryEjLAbzbxtxpJXbdkjc/W51nYaNcAwN9T/eOWWlHrb1PIzqy3p -gtq0ydQ4rdamgUzlXpFH95LXXzAs4D9Z7Xuda9aHctSxSha9V3bGnQ9o2UW9S3lt -fgEMgAGByDvq5yiS0l95d8IvcU38FAUA/ui6Z7ZELD380OF6rq28vFm4xmcCAwEA -AQKCAgA1CAWrPQ9yd1Wqbk9fkYuA4Do9hNfcqDzejY4SuyWjWF0e6i9Y70vZ+gJU -1+IOfURZ0+Y7SX44Y/Vr8xjAU4TsQXl9IhjCONN43pcQJmpGmyxjqtel54r8ARqT -b3NI2fY5WH6t1JVJvAC2UHyCXKgpt4ENEQ5GzGHl7TnfZm2JZb0XBdh2BOrbB2bq -0/2O6gZ2EnNNFYDMY9j2X3i0LoroZ2k/M4B627/R+3D4k0HFCHnAOO709MucqlVX -E0O6GaaDxxTYBQ+5onxPo6Idd/tncjWz1/EIQ1inFJSzvXmZNYEhrovkl84feGCF -cVTuL3E8AzP9fl1eqEBq42wu5RSM+qv+idG/HZXFpUm0+LhTWIYT7Mju+znc31cr -7FUCZz9l41MTWM+4ZjvJysfu77nGiruREybMVFHnDX8WIL5L0IBdWY9oFTZDu2o9 -eg/tVY86xQf4gL1hrtAVy5aAN6Jifmh2PWW3gma6jrxwkZMYSsC150GofD7JP1/d -2+S0bNQ5WsepqemwRkw/AW5ZPbSxUWNGsb2lbQN2y6mj/jN2AZroiUICVyDVnFGR -0bDxEPsHhsj7z5314V9ZuM0XBv1yZTL9v6C8yfuyY7n6K3gtH561v5X1x0GqrMlO -wOS/XMfZWAW3TBMzw/ngE1U3S1CoQ9vL2Lp+JcJVPQxI04H4cQKCAQEA2ODV99xO -IC7e3fNdc8ePM5BCnsy9s7K57MUSl1QGYLu3NP/LPnAXPfbErOQGvPSrC/LxZGdl -9UiwE04ffq6jSj5oMRT0w49d8NZbgscGF3Hth/lkYWMvLr/g/uPeP0IGgYeJklpb -byckxIwPyONb7NfHqLQyt26xrgUsKXbnau8PZRzRukXXgxBGaQPHXKC1EhVRJxHo -osNMJFvwcQsDvoOb2BHf++ZnrHcLbIqrhrCRd2npIsJP+Y24RpKeqf96ORgRFFMx -7HVIpcIZ0xrGWNFllpBKcWW7D3XHriMRgQboiuIXji6sKJMEGBOr8d0Jvh3aN8EG -QAuohOkA8zhd+QKCAQEAwgiLxedWOAHrOzQp9AdDTaO24ZTqYfByFPu7Lyy1QWcK -qjOy1BZ7UZBjLA5N15yncWtaxVtVUqhidlca7rnjNh2KUBBAoP5ysYTZDj4iZiqD -r2qGMs4suXQXpaLhV4oNxLij83oDOFWbyBaT4dhPfu/ink/a/BtLQr7PjBe2QozO -RXl88JhXrFZC3delxLlkWNxFAIROK9Gy+5vS70T3jBbnXV3uYu1GLxRd7ac5wEvf -eJ5fgYIrwpNZhBiEvCGvw4KZEMVlII2Het1VEwlgvwi5BboC1fgvPILuyfaqU1hC -UURVx9kbK+eXKFNTnJmXZ1APcrSL3yR5l8El0gLfXwKCAQB5J6kATPbr7aMc9G4Z -Vrr9wDInShMTht1skm8fUJY7hcID/n2IxM+HEwqt1riY6EmBZFdLkqOXkeKi9jqL -531a5WAJQz603KtK9kSq2wnhIQjjtANKkED0acaEPcGTLww5YgENXCLGBmT405Gh -1LkupyBnczr90/24TDVueHkw+JluD9ptp8SL+BSNZ8oASdq9XmE2ZT/DZTxs91tr -mw6daCIwY1PwAbea1enw2cqWdJ7eybCudUR+EZiG/DNcnlal/HbWFt/Fy4Mo7jTF -BKkXRursYHDlxkHokhBZjEej/7r6kvE7wYD1LaPU23BrreJabYzRJqiaMKkUjqG0 -PaHpAoH/cCyAoekb70UsuQ1vM6T/Q+Nh1whAeF6JlZfyn1M5dpYblEpPGE3m/UB8 -FPwwBHaMVwW5dF9Y9HBkfAXfjCPrN13cmGzM8EYqCHbFBXbmbCon90GeU8DeGlkD -Ur7Xh8RIMXKNVjUzp13WsdZbqDVlv/zvBpckApvE5tFpK5SJye+dUIzrrixkgz1b -auVA8Ho9iSHOwn26s5qH3u43BBzbACI9n7yuFIMYEB+2La5/tKwohZMJo9Wddt+4 -4KVMF3diJaVa2FdGH4uyx1Ut2dOSGt7wGF9xREI3DMSlSB1SWtyrelAHR4I8MyWw -PZ/utpoU7aVbSPUX5+kuYo1HUCJZAoIBADZOSDz8jqumr6Km0n758+7sOGDKhb0Z -ZBbq8hpp9MItj8hGjFGYcpnsErj07kMKIVVAqBqi4ms40Li+HHspWe4HSiZMRDRL -oLDJ64bg7B1NKWmCeU9anForgs8j6ZAxztDk/D3CopXFhglPZtW/viZdOZnvQ1hI -0x98yjE72IAZzvejmJm8XoVDfu4GXLfgZDuxuNNDaLn7huuhnuoxIe3LIgSAq/Gv -t9DnLGiDL880JSi8WazBscSxnZq4r2mG83OobxHKeuQvu+Lsy1bRFgeP5fey3+br -yYGUYK65dBeHnj5KpmQNW+UFP0Ud8rgSdxGc3I6C9OkgtQfd+2jbLxg= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDbyevJD/4RdUvO +nQnKdWc7gSocTWHXrOwZowunekDknOlU9Ly5BHsQixnaa4jtQ5zjtHmgZvfFQDQF +5S0WYqPgNxHlUC0iYl+oVtDolKaZkNN1U8I1CGAh1gAVZcG4yqx0XWh4+tq1kj38 +RfEeSt4dkGXUmsxwNeh309CUpuJHT5SnD7oLDMnd/mjfZGTjR6//uaamXSiGFSlc +IkH1ED+peKgcAGqC6KW+ckHBvCd4Pwzv7mPCDFkOARqiPkUEGidk2J5hxHN35EYj +NechYJ1SQKWQFgTpyJkjMG+YG6Z/SFg6PCZCBIrW2souAIQU1l9FlAOvn0A4Z4TU +pZ1Z3/6hJR0+b0yi8nuy00ZfFQjSWSBiitobWZwIveH+ezG6xWMGtYkHxT71AO1R +9m4lnwXYNwBMNi2F5MRT+7AIeE65RD0P0hZFBL9AQz80F942EgK/djETpTWuUoEa +nAm9SiCfOjCj1OsDpdEyFrBohCOAY99/LB+LohiXpbU8wb6S20qOdhgvJT+UumLI +KRXE00KwL5n/e/RpyaGDJnd87XgkST28lj6hsqF+ToCXK1vN0zaF08d5cUb0lgz8 +As3IrtuWCN6nyHj7rwZJltQjgWh8vgF5ecY4FAtr0DKY9vP2NjVBDhDDAnFv+rDT +X/1Esz6NzmnpT/jGWFpcKSkHybzvOwIDAQABAoICAGvZrrDv/w1UEInnd2l4qNqD +LEXNFwj9WrHN1ycsj3FfKgQ9P+pOL5FToLEUeZFrzy3RXql+PXD2a1DFP5Wve5jH +jSmIaU1pFmWFjGfEoefTMGBvZGofnQSdfVcCosfp9Cv5tg0/k1jr1SzYqVPjCxFY +zQ9BkrYxTcyXDNAVAA5cF6bOyc0oTlp3v8B3BCafA9cpo4QZTIxXD/+c896sRYEE +qwFqltl8DfDhIjFOAdQ4pEGMKjmLU5vSoN4iTj2QE+qhfT6eZsbbJiubBLrVLP2y +qUuF8o1BmRuyZeAvGUl2F40j/OYCUprRq8pdCHMtKnz6/8ALFjlXWs/6ILd8efZy +Yifi751OaCYslNNmEEabpM1Y4X2ey6pGTkd3WNwxIKEsnxMFHpIwjO+wi+ExNTtq +scAfULKV0m+UZcUX7ahMIyWKKEK1QOsvktgrMKR4KgFcB8BP6hvHgYHALLMSwvgo +v7lOE7gSMwIWwEpHZi1etD0FOk0GitMGP/5Q1UW0AWIjjLYL+uucc1Bfrtx5hXWg +4yR1qNcjaVP+g34dUwQ9cz3rxGVfSjcszrOFgYGtTQalIfz1oo8VsiqJoq/PDJ4n +h4IHX46Y23GyNBkLfe2O/lH+w2OOE78h+CEDwef2l72rbAcQEAkLYlE1MqKvmX2H +1ZcwwS1JJxpTZElwg5hJAoIBAQDy70F2dqkH2/mpz2cwbgA2ZU1WD9e/Rv88lVaj +P3cPeneQ/ka2gfOCIPuelQfCkhbWr4vAcT2gpnMEW449JAWdaUJPdiWt9MrrXbDb +m3EstU2H6MvJeiTI183IIQAFUwRK8+VssjAmu8e4JuvozWaQBKqmm1ZZ8ZjyKESY +PrPABmgz2MHN5m/yovtHMGAhYZtrYMIX4r5+qzrMbWCyfjaMOPV5angfGZXKmMoP +DBZGGx+9YS2eXsrdYqBoSMF01/SCQEZ0qcIUc9bsA7dyraD493z7Appr5zIPuYRv +ZqKVt8v2lsoWeBYQtnuAEzivLfQQbHzk2a3EG0EWsrBXSdHlAoIBAQDnm/3OoCVR +EjzWfjRchgXW42siQGKZOzqS0ONwPCxGbofz1H6ePpl1ki8uj4pST6hhij5cnyfj +BeH+jieq8wpf2TXinTfKF8plkI6UKVyx7AlvCKJa+BAewBCgHWlgvnUFan//sXMa +oVFsVFm8CswB1mPy/GahQTphV84VpkCkxjbLDJpWCv/NEUDFRsGdNNsM7ZzgxmWQ +tS/Tq9ZxTidWIEUsloH0ptCiPFq9v8A4vpy8FVJg2HF9nu7HJOTzdx6XXLfNevCm +pdG5yhVnRCbilMjlF7UczrJnEzfsLDgq2FDavZvAgluM7OSy5i/iP1nppurJrDfu +wvuPjEAFZCqfAoIBAFz1vMEOCwRc4p5eJQhdL2uhPASwr0gcbNLL+okhYffkTxiG +lJyGAZ5L7Lz0HjIQIPkp13peWsy8pMqIFlfDDvr7Kkqp1SVJxNRn16NRSe9nuTFG +naSNPsI0AiSQrCddawkbZzE+xjosJPKMwj0lTBfUEVELo/FrB7GyQv1XEQVr7jfY +u+CES1Y3k5vcIQzlAsoGDn+BXo+uCiPrvtLOhSUHLre1TB3qdm0+JiO35lYRTwXy +x7gQYeRxNxxwqsaBVKj623sM8p1M7nMFwQ767dwXKBYNGvYrU/a8cLcIOTK9UCkd +7lrGvPrMOVn7APfXx8r5GJmhymJ+Y1vCKMReFRkCggEARBuTH1fooVaDnW+vZiXj +QbXOo1vbosPQDaxvtWl8e5m1UPaD5XTzG6fJSB/dxSa3lTdZ71K+H/xy4LBc6eBe +hKETP3iMF1m3ovZHdzpSj9+hRo17cNTRqLlCKlN70X8wWk9fJInCF2qozvpfCPUD +2OZ6rdGbPIA58rYnXCZvjATdyAgEupixcFRKf7i25svMPpkBMPgmlc8yKUS24srg +tmr27I212YPUhmDJLlNc61LiNEzw1JnRhedewCERUqcMz+XsBvjIG7lXWxMaCCv6 +Y1btXLEECVyfCcCVCVZXjgHehPSNAKdU1LQ07IbUPevzPMpg8L08FXIxF51eljdG +cQKCAQBUaLWgDVT45v9XKOBqFESipSSdcU8FcdY3BMRBx4Ojfe6Xo6NIN9dNJd2x +QFt/oF48bAh/nHkr365tLrn28Pl8GJFjs7OzQXxoaK3Qf1M+sPptMaGaMLmsdMCw +dyoYjpI67pAk/N0uEhYZGm4XueYZlz8LxxE6O6MuAQixBGBHuBbzC3ujOm7VVmk8 +HLPiuPoQmP9EojGggC+FeYmL3KAkmob7MWNu52AIwJbzl0fWy+Bkc7cQ24nS5PK5 +WQOGXknqzqe0qykgS/V3bJ1CQdqInmhbcELQj32OdlBaTlIIJwBnz/broCZoID25 +PRpZMcWnzreUkNyzlUzMOwEobqEW +-----END PRIVATE KEY----- diff --git a/goproxy/main.go b/goproxy/main.go index 0aa872f..7ecff07 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -120,6 +120,17 @@ func AdBlockMatcherSetBlacklistCallback(callback unsafe.Pointer) { adBlockBlacklistCallback = callback } +//export SetCertificate +func SetCertificate(certFileC *C.char, keyFileC *C.char) int16 { + mainInteropSyncMutex.Lock() + defer mainInteropSyncMutex.Unlock() + + certFile := C.GoString(certFileC) + keyFile := C.GoString(keyFileC) + loadAndSetCa(certFile, keyFile) + return SUCCESS +} + //export StartGoServer func StartGoServer(portHttp, portHttps, portConfigurationServer int16, certFileC *C.char, keyFileC *C.char, bannedImageFileC *C.char) int16 { mainInteropSyncMutex.Lock() From 2382fe5f04c078d1a0adf3cef02c192993fcb9b2 Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Mon, 2 Dec 2024 16:28:18 +0100 Subject: [PATCH 25/27] merge win version fixes --- goproxy/goproxystarter.go | 34 ++++++++++++-------- goproxy/http2.go | 66 ++++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 35 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 5c59296..cff46dd 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -41,7 +41,7 @@ var BLOCKED_IMAGE_BYTES []byte const BLOCKED_IMAGE_CONTENT_TYPE = "image/webp" const MIN_FILTERABLE_LENGTH_IMAGE = 1024 - +const ENABLE_IMAGE_FILTERING = false const DEFAULT_HTTPS_PORT uint16 = 443 type HttpsHandler func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) @@ -329,15 +329,15 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert server = startHttpServer(portHttp) proxy.OnRequest().DoFunc( - func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { userData := make(map[string]interface{}) ctx.UserData = userData monitorLogFileSize() if adBlockMatcher != nil { - category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(r.URL.String(), r.Host, r.Referer()) + category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(req.URL.String(), req.Host, req.Referer()) if category != nil && matchType == Included { - url := r.URL.String() + url := req.URL.String() if adBlockBlacklistCallback != nil { unsafeUrl := C.CString(url) unsafeCategory := C.CString(*category) @@ -348,21 +348,22 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert log.Printf("Page %s blocked by url, category %s", url, *category) - if strings.Contains(r.URL.Host, "vimeo") { - r.Header.Set("cookie", CookiePatchSafeSearch(r.URL.Host, r.Header.Get("cookie"))) + if strings.Contains(req.URL.Host, "vimeo") { + req.Header.Set("cookie", CookiePatchSafeSearch(req.URL.Host, req.Header.Get("cookie"))) } - - r.URL.RawPath = HostPathForceSafeSearch(r.URL.Host, r.URL.RawPath) - return r, goproxy.NewResponse(r, + response := NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy)) + + req.URL.RawPath = HostPathForceSafeSearch(req.URL.Host, req.URL.RawPath) + return req, response } } // if strings.Contains(r.Host, "yandex") { // return r, goproxy.NewResponse(r, "text/html; charset=UTF-8", 200, "Blocked by rules1") // } - return r, nil + return req, nil }) proxy.OnResponse().DoFunc( @@ -387,7 +388,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert if !isCertInException(certThumbPrint) { message := adBlockMatcher.GetBadCertPage(ctx.Req.URL.String(), ctx.Req.URL.Host, certThumbPrint) - return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + return NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) } } } @@ -415,7 +416,7 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert if isAllowed { return resp } else { - return goproxy.NewResponse(resp.Request, BLOCKED_IMAGE_CONTENT_TYPE, 200, string(BLOCKED_IMAGE_BYTES)) + return NewResponse(resp.Request, BLOCKED_IMAGE_CONTENT_TYPE, 200, string(BLOCKED_IMAGE_BYTES)) } } } else { @@ -430,7 +431,8 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert if category != nil { log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", resp.Request.URL.String(), *category, strings.Join(matches, ", ")) message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false) - return goproxy.NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + blockedResp := NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) + return blockedResp } } return resp @@ -444,6 +446,12 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert } } +func NewResponse(r *http.Request, contentType string, status int, body string) *http.Response { + resp := goproxy.NewResponse(r, contentType, status, body) + resp.Header.Add("Access-Control-Allow-Origin", "*") + return resp +} + func stopGoProxyServer() { if server != nil { context, _ := context.WithTimeout(context.Background(), 1*time.Millisecond) diff --git a/goproxy/http2.go b/goproxy/http2.go index fe48851..ac203b3 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -11,6 +11,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/cloudveiltech/goproxy" "golang.org/x/net/http2" @@ -39,10 +40,17 @@ type Http2Handler struct { debouncers map[uint32]func(f func()) connectionReadyForData bool rwMutex *sync.RWMutex + verbose bool + id int64 } func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UConn) bool { log.Print("Running http2 handler for " + r.URL.String()) + verbose := false + /* if strings.Contains(r.URL.String(), "google.com") || strings.Contains(r.URL.String(), "gstatic") { + log.Printf("Google %s: serveHttp2Filtering", r.URL.String()) + verbose = true + }*/ http2Handler := &Http2Handler{ maxFrameSize: 1024, @@ -55,6 +63,8 @@ func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UC debouncers: make(map[uint32]func(f func())), connectionReadyForData: false, rwMutex: &sync.RWMutex{}, + verbose: verbose, + id: time.Now().Unix(), } go func() { http2Handler.processHttp2Stream(rawClientTls, remote) @@ -71,7 +81,7 @@ func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tl return } if string(b) != preface { - log.Printf("ReadFrame: preface error") + log.Printf("%d ReadFrame: preface error, flagged: %v", http2Handler.id, http2Handler.verbose) return } remote.Write(b) @@ -83,15 +93,16 @@ func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tl go func() { decoder := hpack.NewDecoder(65536, nil) for { - res := http2Handler.readFrame(reverseFramer, directFramer, decoder, false) + res := http2Handler.readFrame(reverseFramer, directFramer, decoder, remote.ConnectionState(), false) if res != STATUS_OK { + local.Close() return } } }() decoder := hpack.NewDecoder(65536, nil) for { - res := http2Handler.readFrame(directFramer, reverseFramer, decoder, true) + res := http2Handler.readFrame(directFramer, reverseFramer, decoder, remote.ConnectionState(), true) if res != STATUS_OK { if res == STATUS_BLOCKED { remote.Close() @@ -101,31 +112,40 @@ func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tl } } -func isContentTypeFilterable(contentType string) bool { +func isContentTypeFilterable(contentType string, contentLength int64) bool { + if contentLength > MAX_FILTERABLE_LENGTH { + return false + } if strings.Contains(contentType, "protobuf") { return false } - return strings.Contains(contentType, "html") || - strings.Contains(contentType, "json") || - strings.Contains(contentType, "image/png") || - strings.Contains(contentType, "image/jpg") || - strings.Contains(contentType, "image/jpeg") || - strings.Contains(contentType, "image/webp") + + result := strings.Contains(contentType, "html") || strings.Contains(contentType, "json") + if ENABLE_IMAGE_FILTERING && !result { + result = result || + strings.Contains(contentType, "image/png") || + strings.Contains(contentType, "image/jpg") || + strings.Contains(contentType, "image/jpeg") || + strings.Contains(contentType, "image/webp") + } + return result } -func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) int { +func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, connectionState tls.ConnectionState, client bool) int { f, err := directFramer.ReadFrame() if err != nil { - log.Printf("ReadFrame client %v, err: %v", client, err) + log.Printf("%d ReadFrame client %v, err: %v, flagged: %v", http2Handler.id, client, err, http2Handler.verbose) return STATUS_ENDED } - switch f.Header().Type { case http2.FrameData: fr := f.(*http2.DataFrame) body := fr.Data() streamId := f.Header().StreamID + if http2Handler.verbose { + log.Printf("%d Data frame received client: %v, flagged: %v", http2Handler.id, client, http2Handler.verbose) + } http2Handler.rwMutex.RLock() lastHttpResponse := http2Handler.lastHttpResponse[streamId] @@ -155,16 +175,16 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F lastHttpResponse = http2Handler.lastHttpResponse[streamId] bodyChunks = http2Handler.responseBodyMapChunks[streamId] http2Handler.rwMutex.RUnlock() - + if http2Handler.verbose && fr.StreamEnded() { + log.Printf("%d Stream ended, flagged: %v", http2Handler.id, client, http2Handler.verbose) + } streamEnded := fr.StreamEnded() || force if !whitelisted && lastHttpResponse != nil && !client { contentType := lastHttpResponse.Header.Get("Content-Type") - isContentTypeFilterable := isContentTypeFilterable(contentType) - isImage := strings.Contains(contentType, "image") - putResponseBody(bodyChunks, lastHttpResponse) contentLength := lastHttpResponse.ContentLength + isContentTypeFilterable := isContentTypeFilterable(contentType, contentLength) + putResponseBody(bodyChunks, lastHttpResponse) - isContentTypeFilterable = isContentTypeFilterable && (contentLength < MAX_FILTERABLE_LENGTH || isImage) if isContentTypeFilterable && streamEnded { if contentLength > MIN_FILTERABLE_LENGTH { http2Handler.rwMutex.RLock() @@ -259,7 +279,7 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F writeHeadersImmediately := whitelisted || client || fr.StreamEnded() if client { request := makeHttpRequest(nil, headerFields) - var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1)} + var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1), ConnectionState: &connectionState} http2Handler.rwMutex.Lock() http2Handler.lastHttpRequest[streamId] = request @@ -288,15 +308,17 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F } } else { response := makeHttpResponse(nil, headerFields) - + if http2Handler.verbose { + log.Printf("%d Headers received, %d, flagged: %v", http2Handler.id, response.StatusCode, http2Handler.verbose) + } http2Handler.rwMutex.Lock() http2Handler.lastHttpResponse[streamId] = response http2Handler.lastHttpResponse[streamId].Request = http2Handler.lastHttpRequest[streamId] http2Handler.rwMutex.Unlock() contentType := response.Header.Get("Content-Type") - // contentLength, _ := strconv.Atoi(response.Header.Get("Content-Length")) - if !isContentTypeFilterable(contentType) { + contentLength, _ := strconv.ParseInt(response.Header.Get("Content-Length"), 0, 64) + if !isContentTypeFilterable(contentType, contentLength) { writeHeadersImmediately = true } } From 89a7ad170d15fb054b3bc70bd7c850136e8e343c Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 6 Dec 2024 12:56:45 +0100 Subject: [PATCH 26/27] updated http2 handler --- goproxy/goproxystarter.go | 6 + goproxy/http2.go | 1280 +++++++++++++++++++------------------ 2 files changed, 657 insertions(+), 629 deletions(-) diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index cff46dd..934b020 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -333,6 +333,12 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert userData := make(map[string]interface{}) ctx.UserData = userData + contentLength, _ := strconv.ParseInt(req.Header.Get("Content-Length"), 0, 64) + if contentLength > MIN_FILTERABLE_LENGTH { + userData["blocked"] = false + return req, nil + } + monitorLogFileSize() if adBlockMatcher != nil { category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(req.URL.String(), req.Host, req.Referer()) diff --git a/goproxy/http2.go b/goproxy/http2.go index ac203b3..482a806 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -1,629 +1,651 @@ -package main - -import ( - "bytes" - "io" - "io/ioutil" - "log" - "net/http" - "net/url" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/cloudveiltech/goproxy" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" - - tls "github.com/refraction-networking/utls" -) - -var http2ProxySessionCounter int64 - -const MAX_FILTERABLE_LENGTH = 1024 * 1024 -const MIN_FILTERABLE_LENGTH = 100 - -const STATUS_BLOCKED = 0 -const STATUS_OK = 1 -const STATUS_ENDED = 2 - -type Http2Handler struct { - maxFrameSize uint32 - lastHttpResponse map[uint32]*http.Response - lastHttpRequest map[uint32]*http.Request - lastHeadersBlock map[uint32]*http2.HeadersFrameParam - proxyCtx map[uint32]*goproxy.ProxyCtx - lastHeadersMap map[uint32][]hpack.HeaderField - responseBodyMapChunks map[uint32][][]byte - debouncers map[uint32]func(f func()) - connectionReadyForData bool - rwMutex *sync.RWMutex - verbose bool - id int64 -} - -func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UConn) bool { - log.Print("Running http2 handler for " + r.URL.String()) - verbose := false - /* if strings.Contains(r.URL.String(), "google.com") || strings.Contains(r.URL.String(), "gstatic") { - log.Printf("Google %s: serveHttp2Filtering", r.URL.String()) - verbose = true - }*/ - - http2Handler := &Http2Handler{ - maxFrameSize: 1024, - lastHttpResponse: make(map[uint32]*http.Response), - lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), - lastHeadersMap: make(map[uint32][]hpack.HeaderField), - lastHttpRequest: make(map[uint32]*http.Request), - proxyCtx: make(map[uint32]*goproxy.ProxyCtx), - responseBodyMapChunks: make(map[uint32][][]byte), - debouncers: make(map[uint32]func(f func())), - connectionReadyForData: false, - rwMutex: &sync.RWMutex{}, - verbose: verbose, - id: time.Now().Unix(), - } - go func() { - http2Handler.processHttp2Stream(rawClientTls, remote) - }() - - return true -} - -func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tls.UConn) { - const preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - b := make([]byte, len(preface)) - if _, err := io.ReadFull(local, b); err != nil { - log.Printf("ReadFrame: preface %v", err) - return - } - if string(b) != preface { - log.Printf("%d ReadFrame: preface error, flagged: %v", http2Handler.id, http2Handler.verbose) - return - } - remote.Write(b) - - http2.VerboseLogs = false - directFramer := http2.NewFramer(remote, local) - reverseFramer := http2.NewFramer(local, remote) - - go func() { - decoder := hpack.NewDecoder(65536, nil) - for { - res := http2Handler.readFrame(reverseFramer, directFramer, decoder, remote.ConnectionState(), false) - if res != STATUS_OK { - local.Close() - return - } - } - }() - decoder := hpack.NewDecoder(65536, nil) - for { - res := http2Handler.readFrame(directFramer, reverseFramer, decoder, remote.ConnectionState(), true) - if res != STATUS_OK { - if res == STATUS_BLOCKED { - remote.Close() - } - return - } - } -} - -func isContentTypeFilterable(contentType string, contentLength int64) bool { - if contentLength > MAX_FILTERABLE_LENGTH { - return false - } - if strings.Contains(contentType, "protobuf") { - return false - } - - result := strings.Contains(contentType, "html") || strings.Contains(contentType, "json") - if ENABLE_IMAGE_FILTERING && !result { - result = result || - strings.Contains(contentType, "image/png") || - strings.Contains(contentType, "image/jpg") || - strings.Contains(contentType, "image/jpeg") || - strings.Contains(contentType, "image/webp") - } - return result -} - -func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, connectionState tls.ConnectionState, client bool) int { - f, err := directFramer.ReadFrame() - if err != nil { - log.Printf("%d ReadFrame client %v, err: %v, flagged: %v", http2Handler.id, client, err, http2Handler.verbose) - return STATUS_ENDED - } - switch f.Header().Type { - case http2.FrameData: - fr := f.(*http2.DataFrame) - body := fr.Data() - - streamId := f.Header().StreamID - if http2Handler.verbose { - log.Printf("%d Data frame received client: %v, flagged: %v", http2Handler.id, client, http2Handler.verbose) - } - - http2Handler.rwMutex.RLock() - lastHttpResponse := http2Handler.lastHttpResponse[streamId] - bodyChunks := http2Handler.responseBodyMapChunks[streamId] - ctx := http2Handler.proxyCtx[streamId] - http2Handler.rwMutex.RUnlock() - - chunk := make([]byte, len(body)) - copy(chunk, body) - bodyChunks = append(bodyChunks, chunk) - - http2Handler.rwMutex.Lock() - http2Handler.responseBodyMapChunks[streamId] = bodyChunks - http2Handler.rwMutex.Unlock() - - whitelisted := false - if ctx != nil && ctx.UserData != nil { - blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] - whitelisted = exists && !(blocked.(bool)) - } - - processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) int { - if force { - log.Print("Force ending stream on timeout") - } - http2Handler.rwMutex.RLock() - lastHttpResponse = http2Handler.lastHttpResponse[streamId] - bodyChunks = http2Handler.responseBodyMapChunks[streamId] - http2Handler.rwMutex.RUnlock() - if http2Handler.verbose && fr.StreamEnded() { - log.Printf("%d Stream ended, flagged: %v", http2Handler.id, client, http2Handler.verbose) - } - streamEnded := fr.StreamEnded() || force - if !whitelisted && lastHttpResponse != nil && !client { - contentType := lastHttpResponse.Header.Get("Content-Type") - contentLength := lastHttpResponse.ContentLength - isContentTypeFilterable := isContentTypeFilterable(contentType, contentLength) - putResponseBody(bodyChunks, lastHttpResponse) - - if isContentTypeFilterable && streamEnded { - if contentLength > MIN_FILTERABLE_LENGTH { - http2Handler.rwMutex.RLock() - ctx := http2Handler.proxyCtx[streamId] - http2Handler.rwMutex.RUnlock() - resp := proxy.FilterResponse(lastHttpResponse, ctx) - - if resp != lastHttpResponse { //new response - if !http2Handler.connectionReadyForData { - reverseFramer.WriteSettings() - } - writeHeaders(directFramer, &http2.HeadersFrameParam{ - StreamID: streamId, - BlockFragment: encodeHeaders(resp), - EndStream: false, - EndHeaders: true, - PadLength: 0, - Priority: http2.PriorityParam{}, - }, decoder) - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - writeFinalData(directFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) - // directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) - http2Handler.rwMutex.Lock() - delete(http2Handler.lastHttpResponse, streamId) - delete(http2Handler.lastHttpRequest, streamId) - delete(http2Handler.responseBodyMapChunks, streamId) - http2Handler.rwMutex.Unlock() - return STATUS_BLOCKED - } - } - } else if isContentTypeFilterable { - return STATUS_OK - } - } - - http2Handler.rwMutex.RLock() - header, ok := http2Handler.lastHeadersBlock[streamId] - http2Handler.rwMutex.RUnlock() - if ok { - // headerFields, _ := http2Handler.lastHeadersMap[streamId] - header.EndStream = false - // header.BlockFragment = encodeHeaderFields(headerFields) - writeHeaders(directFramer, header, decoder) - - http2Handler.rwMutex.Lock() - delete(http2Handler.lastHeadersBlock, streamId) - delete(http2Handler.lastHeadersMap, streamId) - http2Handler.rwMutex.Unlock() - } - - for i, _ := range bodyChunks { - streamEnd := i == len(bodyChunks)-1 && streamEnded - directFramer.WriteData(streamId, streamEnd, bodyChunks[i]) - } - - http2Handler.rwMutex.Lock() - delete(http2Handler.responseBodyMapChunks, streamId) - http2Handler.rwMutex.Unlock() - return STATUS_OK - } - - processDataFrameFunc(false, streamId, directFramer, reverseFramer, decoder, client) - /* - debouncer, exists := http2Handler.debouncers[streamId] - if !exists { - debouncer = debounce.New(time.Millisecond * 1000) - http2Handler.debouncers[streamId] = debouncer - } - debouncer(func() { - _, exists := http2Handler.debouncers[streamId] - if exists { - processDataFrameFunc(true, streamId, directFramer, reverseFramer, decoder, client) - } - })*/ - case http2.FrameHeaders: - fr := f.(*http2.HeadersFrame) - - headerFields, _ := decodeAllHeaders(directFramer, fr, decoder) - if len(headerFields) == 0 { - log.Printf("Error parsing headers") - } - whitelisted := false - - streamId := f.Header().StreamID - ctx := http2Handler.proxyCtx[streamId] - if ctx != nil && ctx.UserData != nil { - blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] - whitelisted = exists && !(blocked.(bool)) - } - - writeHeadersImmediately := whitelisted || client || fr.StreamEnded() - if client { - request := makeHttpRequest(nil, headerFields) - var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1), ConnectionState: &connectionState} - - http2Handler.rwMutex.Lock() - http2Handler.lastHttpRequest[streamId] = request - http2Handler.proxyCtx[streamId] = ctx - http2Handler.rwMutex.Unlock() - _, resp := proxy.FilterRequest(request, ctx) - - if resp != nil { - if !http2Handler.connectionReadyForData { - reverseFramer.WriteSettings() - } - writeHeaders(reverseFramer, &http2.HeadersFrameParam{ - StreamID: streamId, - BlockFragment: encodeHeaders(resp), - EndStream: false, - EndHeaders: true, - PadLength: 0, - Priority: fr.Priority, - }, decoder) - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - - writeFinalData(reverseFramer, streamId, buf, int(http2Handler.maxFrameSize)) - reverseFramer.WriteGoAway(streamId, http2.ErrCodeRefusedStream, nil) - return STATUS_BLOCKED - } - } else { - response := makeHttpResponse(nil, headerFields) - if http2Handler.verbose { - log.Printf("%d Headers received, %d, flagged: %v", http2Handler.id, response.StatusCode, http2Handler.verbose) - } - http2Handler.rwMutex.Lock() - http2Handler.lastHttpResponse[streamId] = response - http2Handler.lastHttpResponse[streamId].Request = http2Handler.lastHttpRequest[streamId] - http2Handler.rwMutex.Unlock() - - contentType := response.Header.Get("Content-Type") - contentLength, _ := strconv.ParseInt(response.Header.Get("Content-Length"), 0, 64) - if !isContentTypeFilterable(contentType, contentLength) { - writeHeadersImmediately = true - } - } - - header := http2.HeadersFrameParam{ - StreamID: streamId, - BlockFragment: encodeHeaderFields(headerFields), - EndStream: fr.StreamEnded(), - EndHeaders: fr.HeadersEnded(), - PadLength: 0, - Priority: fr.Priority, - } - - if writeHeadersImmediately { - writeHeaders(directFramer, &header, decoder) - } else { - http2Handler.rwMutex.Lock() - http2Handler.lastHeadersMap[f.Header().StreamID] = headerFields - http2Handler.lastHeadersBlock[f.Header().StreamID] = &header - http2Handler.rwMutex.Unlock() - } - case http2.FramePriority: - fr := f.(*http2.PriorityFrame) - directFramer.WritePriority(f.Header().StreamID, fr.PriorityParam) - case http2.FrameRSTStream: - fr := f.(*http2.RSTStreamFrame) - directFramer.WriteRSTStream(f.Header().StreamID, fr.ErrCode) - case http2.FrameSettings: - fr := f.(*http2.SettingsFrame) - if !client { - http2Handler.connectionReadyForData = true //once server sent the settings we're good to go - } - - if fr.IsAck() { - directFramer.WriteSettingsAck() - } else { - params := make([]http2.Setting, 0) - for i := 0; i < fr.NumSettings(); i++ { - setting := fr.Setting(i) - params = append(params, setting) - if setting.ID == http2.SettingHeaderTableSize && client { - decoder.SetMaxDynamicTableSize(setting.Val) - } - if setting.ID == http2.SettingMaxFrameSize && client { - if http2Handler.maxFrameSize > setting.Val { - http2Handler.maxFrameSize = setting.Val - } - } - - } - directFramer.WriteSettings(params...) - } - - case http2.FramePushPromise: - fr := f.(*http2.PushPromiseFrame) - directFramer.WritePushPromise(http2.PushPromiseParam{ - - StreamID: f.Header().StreamID, - PromiseID: fr.PromiseID, - BlockFragment: fr.HeaderBlockFragment(), - EndHeaders: fr.HeadersEnded(), - PadLength: 0, - }) - case http2.FramePing: - fr := f.(*http2.PingFrame) - directFramer.WritePing(fr.IsAck(), fr.Data) - case http2.FrameGoAway: - fr := f.(*http2.GoAwayFrame) - directFramer.WriteGoAway(fr.LastStreamID, fr.ErrCode, fr.DebugData()) - case http2.FrameWindowUpdate: - fr := f.(*http2.WindowUpdateFrame) - directFramer.WriteWindowUpdate(f.Header().StreamID, fr.Increment) - case http2.FrameContinuation: - fr := f.(*http2.ContinuationFrame) - directFramer.WriteContinuation(f.Header().StreamID, fr.HeadersEnded(), fr.HeaderBlockFragment()) - default: - fr := f.(*http2.UnknownFrame) - directFramer.WriteRawFrame(f.Header().Type, f.Header().Flags, f.Header().StreamID, fr.Payload()) - } - - return STATUS_OK -} - -func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpack.Decoder) ([]hpack.HeaderField, []byte) { - buf := new(bytes.Buffer) - res := make([]hpack.HeaderField, 0) - - hostIndex := 0 - pathIndex := 0 - cookieIndex := 0 - decoder.SetEmitEnabled(true) - decoder.SetMaxStringLength(16 << 20) - decoder.SetEmitFunc(func(hf hpack.HeaderField) { - if len(hf.Name) > 0 { - if hf.Name == ":path" { - pathIndex = len(res) - } else if hf.Name == ":authority" { - hostIndex = len(res) - } else if hf.Name == "cookie" { - cookieIndex = len(res) - } - res = append(res, hf) - } - }) - defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) - defer decoder.Close() - - buf.Write(fr.HeaderBlockFragment()) - _, err := decoder.Write(fr.HeaderBlockFragment()) - if err != nil { - log.Printf("Error decode %v", err) - } - if fr.HeadersEnded() { - if hostIndex > 0 || pathIndex > 0 { - res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) - - if cookieIndex > 0 { - res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) - } - } - - return res, buf.Bytes() - } - for { - if f, err := framer.ReadFrame(); err != nil { - break - } else { - continuationFrame := f.(*http2.ContinuationFrame) // guaranteed by checkFrameOrder - buf.Write(continuationFrame.HeaderBlockFragment()) - _, err = decoder.Write(continuationFrame.HeaderBlockFragment()) - if err != nil { - log.Printf("Error decode %v", err) - } - if continuationFrame.HeadersEnded() { - break - } - } - } - - if hostIndex > 0 || pathIndex > 0 { - res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) - if cookieIndex > 0 { - res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) - } - } - - return res, buf.Bytes() -} - -func writeFinalData(framer *http2.Framer, streamId uint32, data *bytes.Buffer, chunkSize int) { - dataToSend := data.Bytes() - for i := 0; i < len(dataToSend); i += chunkSize { - end := i + chunkSize - dataEnded := false - if end >= len(dataToSend) { - end = len(dataToSend) - dataEnded = true - } - - framer.WriteData(streamId, dataEnded, dataToSend[i:end]) - } -} - -func writeHeaders(framer *http2.Framer, param *http2.HeadersFrameParam, decoder *hpack.Decoder) { - dataToSend := param.BlockFragment - chunkSize := 15 * 1024 - for i := 0; i < len(dataToSend); i += chunkSize { - end := i + chunkSize - headesEnd := false - if end >= len(dataToSend) { - end = len(dataToSend) - headesEnd = true - } - - if i == 0 { - /* decoder.SetEmitEnabled(true) - decoder.SetMaxStringLength(16 << 20) - decoder.SetEmitFunc(func(hf hpack.HeaderField) { - if len(hf.Name) > 0 { - log.Printf("Writing header id:%d, %s:%s", param.StreamID, hf.Name, hf.Value) - } - }) - defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) - defer decoder.Close() - - decoder.Write(dataToSend[i:end])*/ - - framer.WriteHeaders(http2.HeadersFrameParam{ - StreamID: param.StreamID, - BlockFragment: dataToSend[i:end], - EndStream: headesEnd && param.EndStream, - EndHeaders: headesEnd, - PadLength: 0, - Priority: param.Priority, - }) - } else { - framer.WriteContinuation(param.StreamID, headesEnd, dataToSend[i:end]) - } - } -} - -func makeHttpRequest(body []byte, header []hpack.HeaderField) *http.Request { - req := http.Request{} - req.Proto = "http/2" - req.ProtoMajor = 2 - req.ProtoMinor = 0 - - req.Header = http.Header{} - scheme := "https" - authority := "" - path := "" - for _, v := range header { - if !strings.HasPrefix(v.Name, ":") { - req.Header.Add(v.Name, v.Value) - } else if v.Name == ":scheme" { - scheme = v.Value - } else if v.Name == ":authority" { - authority = v.Value - } else if v.Name == ":path" { - path = v.Value - } else if v.Name == ":method" { - req.Method = v.Value - } - } - - req.RequestURI = scheme + "://" + authority + path - req.URL, _ = url.ParseRequestURI(req.RequestURI) - req.Host = req.URL.Host - - if len(body) == 0 { - req.Body = http.NoBody - } else { - req.Body = ioutil.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - } - return &req -} - -func makeHttpResponse(body []byte, header []hpack.HeaderField) *http.Response { - resp := http.Response{} - - resp.Proto = "http/2" - resp.ProtoMajor = 2 - resp.ProtoMinor = 0 - resp.Header = http.Header{} - for _, v := range header { - if !strings.HasPrefix(v.Name, ":") { - resp.Header.Add(v.Name, v.Value) - } else if v.Name == ":status" { - resp.Status = v.Value - resp.StatusCode, _ = strconv.Atoi(v.Value) - } - } - if len(body) == 0 { - resp.Body = http.NoBody - resp.ContentLength = 0 - } else { - resp.Body = ioutil.NopCloser(bytes.NewReader(body)) - resp.ContentLength = int64(len(body)) - } - return &resp -} - -func putResponseBody(bodyParts [][]byte, resp *http.Response) { - if len(bodyParts) == 0 { - resp.Body = http.NoBody - } else { - body := make([]byte, 0) - for _, b := range bodyParts { - body = append(body, b...) - } - resp.Body = ioutil.NopCloser(bytes.NewReader(body)) - resp.ContentLength = int64(len(body)) - } -} - -func encodeHeaderFields(fields []hpack.HeaderField) []byte { - buf := new(bytes.Buffer) - encoder := hpack.NewEncoder(buf) - encoder.SetMaxDynamicTableSizeLimit(65536) - buf.Reset() - - for i := 0; i < len(fields); i++ { - encoder.WriteField(fields[i]) - } - return buf.Bytes() -} - -func encodeHeaders(resp *http.Response) []byte { - buf := new(bytes.Buffer) - encoder := hpack.NewEncoder(buf) - // encoder.SetMaxDynamicTableSize(65536) - buf.Reset() - - writeHeader(encoder, ":status", strconv.Itoa(resp.StatusCode)) - writeHeader(encoder, "content-length", strconv.FormatInt(resp.ContentLength, 10)) - for k, vv := range resp.Header { - lowKey := strings.ToLower(k) - for _, v := range vv { - writeHeader(encoder, lowKey, v) - } - } - return buf.Bytes() -} - -func writeHeader(encoder *hpack.Encoder, name, value string) { - encoder.WriteField(hpack.HeaderField{Name: name, Value: value}) -} +package main + +import ( + "bytes" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/cloudveiltech/goproxy" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + + tls "github.com/refraction-networking/utls" +) + +var http2ProxySessionCounter int64 + +const MAX_FILTERABLE_LENGTH = 1024 * 1024 +const MIN_FILTERABLE_LENGTH = 100 + +const STATUS_BLOCKED = 0 +const STATUS_OK = 1 +const STATUS_ENDED = 2 + +type Http2Handler struct { + maxFrameSize uint32 + lastHttpResponse map[uint32]*http.Response + lastHttpRequest map[uint32]*http.Request + lastHeadersBlock map[uint32]*http2.HeadersFrameParam + proxyCtx map[uint32]*goproxy.ProxyCtx + lastHeadersMap map[uint32][]hpack.HeaderField + responseBodyMapChunks map[uint32][][]byte + debouncers map[uint32]func(f func()) + connectionReadyForData bool + rwMutex *sync.RWMutex + verbose bool + id int64 +} + +func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UConn) bool { + log.Print("Running http2 handler for " + r.URL.String()) + verbose := false + // if strings.Contains(r.URL.String(), "womenshealthmag") { + // log.Printf("Google %s: serveHttp2Filtering", r.URL.String()) + // verbose = true + // } + + http2Handler := &Http2Handler{ + maxFrameSize: 1024, + lastHttpResponse: make(map[uint32]*http.Response), + lastHeadersBlock: make(map[uint32]*http2.HeadersFrameParam), + lastHeadersMap: make(map[uint32][]hpack.HeaderField), + lastHttpRequest: make(map[uint32]*http.Request), + proxyCtx: make(map[uint32]*goproxy.ProxyCtx), + responseBodyMapChunks: make(map[uint32][][]byte), + debouncers: make(map[uint32]func(f func())), + connectionReadyForData: false, + rwMutex: &sync.RWMutex{}, + verbose: verbose, + id: time.Now().Unix(), + } + go func() { + http2Handler.processHttp2Stream(rawClientTls, remote) + }() + + return true +} + +func (http2Handler *Http2Handler) processHttp2Stream(local *tls.Conn, remote *tls.UConn) { + const preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + b := make([]byte, len(preface)) + if _, err := io.ReadFull(local, b); err != nil { + log.Printf("ReadFrame: preface %v", err) + return + } + if string(b) != preface { + log.Printf("%d ReadFrame: preface error, flagged: %v", http2Handler.id, http2Handler.verbose) + return + } + remote.Write(b) + + http2.VerboseLogs = false + directFramer := http2.NewFramer(remote, local) + reverseFramer := http2.NewFramer(local, remote) + + go func() { + decoder := hpack.NewDecoder(65536, nil) + for { + res := http2Handler.readFrame(reverseFramer, directFramer, decoder, remote.ConnectionState(), false) + if res != STATUS_OK { + local.Close() + return + } + } + }() + decoder := hpack.NewDecoder(65536, nil) + for { + res := http2Handler.readFrame(directFramer, reverseFramer, decoder, remote.ConnectionState(), true) + if res != STATUS_OK { + if res == STATUS_BLOCKED { + remote.Close() + } + return + } + } +} + +func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, connectionState tls.ConnectionState, client bool) int { + f, err := directFramer.ReadFrame() + if err != nil { + log.Printf("%d ReadFrame client %v, err: %v, flagged: %v", http2Handler.id, client, err, http2Handler.verbose) + return STATUS_ENDED + } + if http2Handler.verbose { + log.Printf("%d Frame received %v client: %v, flagged: %v", http2Handler.id, f.Header().Type, client, http2Handler.verbose) + } + switch f.Header().Type { + case http2.FrameData: + fr := f.(*http2.DataFrame) + body := fr.Data() + + streamId := f.Header().StreamID + if http2Handler.verbose { + log.Printf("%d Data frame received client: %v, flagged: %v", http2Handler.id, client, http2Handler.verbose) + } + + http2Handler.rwMutex.RLock() + lastHttpResponse := http2Handler.lastHttpResponse[streamId] + bodyChunks := http2Handler.responseBodyMapChunks[streamId] + ctx := http2Handler.proxyCtx[streamId] + http2Handler.rwMutex.RUnlock() + + chunk := make([]byte, len(body)) + copy(chunk, body) + bodyChunks = append(bodyChunks, chunk) + + http2Handler.rwMutex.Lock() + http2Handler.responseBodyMapChunks[streamId] = bodyChunks + http2Handler.rwMutex.Unlock() + + whitelisted := false + if ctx != nil && ctx.UserData != nil { + blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] + whitelisted = exists && !(blocked.(bool)) + } + + processDataFrameFunc := func(force bool, streamId uint32, directFramer, reverseFramer *http2.Framer, decoder *hpack.Decoder, client bool) int { + if force { + log.Print("Force ending stream on timeout") + } + http2Handler.rwMutex.RLock() + lastHttpResponse = http2Handler.lastHttpResponse[streamId] + bodyChunks = http2Handler.responseBodyMapChunks[streamId] + http2Handler.rwMutex.RUnlock() + if http2Handler.verbose && fr.StreamEnded() { + log.Printf("%d Stream ended, flagged: %v", http2Handler.id, client, http2Handler.verbose) + } + streamEnded := fr.StreamEnded() || force + if !whitelisted && lastHttpResponse != nil && !client { + contentType := lastHttpResponse.Header.Get("Content-Type") + contentLength := lastHttpResponse.ContentLength + isContentTypeFilterable := isContentTypeFilterable(contentType, contentLength) + putResponseBody(bodyChunks, lastHttpResponse) + + if isContentTypeFilterable && streamEnded { + if contentLength > MIN_FILTERABLE_LENGTH { + http2Handler.rwMutex.RLock() + ctx := http2Handler.proxyCtx[streamId] + http2Handler.rwMutex.RUnlock() + resp := proxy.FilterResponse(lastHttpResponse, ctx) + + if resp != lastHttpResponse { //new response + if !http2Handler.connectionReadyForData { + reverseFramer.WriteSettings() + } + writeHeaders(directFramer, &http2.HeadersFrameParam{ + StreamID: streamId, + BlockFragment: encodeHeaders(resp), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: http2.PriorityParam{}, + }, decoder) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + writeFinalData(directFramer, f.Header().StreamID, buf, int(http2Handler.maxFrameSize)) + // directFramer.WriteGoAway(streamId, http2.ErrCodeCancel, nil) + http2Handler.rwMutex.Lock() + delete(http2Handler.lastHttpResponse, streamId) + delete(http2Handler.lastHttpRequest, streamId) + delete(http2Handler.responseBodyMapChunks, streamId) + http2Handler.rwMutex.Unlock() + return STATUS_BLOCKED + } + } + } else if isContentTypeFilterable { + return STATUS_OK + } + } + + http2Handler.writeDataAndHeaders(decoder, directFramer, f.Header().StreamID, streamEnded) + return STATUS_OK + } + + processDataFrameFunc(false, streamId, directFramer, reverseFramer, decoder, client) + /* + debouncer, exists := http2Handler.debouncers[streamId] + if !exists { + debouncer = debounce.New(time.Millisecond * 1000) + http2Handler.debouncers[streamId] = debouncer + } + debouncer(func() { + _, exists := http2Handler.debouncers[streamId] + if exists { + processDataFrameFunc(true, streamId, directFramer, reverseFramer, decoder, client) + } + })*/ + case http2.FrameHeaders: + fr := f.(*http2.HeadersFrame) + http2Handler.rwMutex.Lock() + headerFields := http2Handler.lastHeadersMap[f.Header().StreamID] + http2Handler.rwMutex.Unlock() + newHeaders, _ := decodeAllHeaders(directFramer, fr, decoder) + headerFields = append(headerFields, newHeaders...) + + if len(headerFields) == 0 { + log.Printf("Error parsing headers") + } + whitelisted := false + + streamId := f.Header().StreamID + ctx := http2Handler.proxyCtx[streamId] + if ctx != nil && ctx.UserData != nil { + blocked, exists := ctx.UserData.(map[string]interface{})["blocked"] + whitelisted = exists && !(blocked.(bool)) + } + + writeHeadersImmediately := whitelisted || client || fr.StreamEnded() + if client { + request := makeHttpRequest(nil, headerFields) + var ctx = &goproxy.ProxyCtx{Req: request, Session: atomic.AddInt64(&http2ProxySessionCounter, 1), ConnectionState: &connectionState} + + http2Handler.rwMutex.Lock() + http2Handler.lastHttpRequest[streamId] = request + http2Handler.proxyCtx[streamId] = ctx + http2Handler.rwMutex.Unlock() + _, resp := proxy.FilterRequest(request, ctx) + + if resp != nil { + if !http2Handler.connectionReadyForData { + reverseFramer.WriteSettings() + } + writeHeaders(reverseFramer, &http2.HeadersFrameParam{ + StreamID: streamId, + BlockFragment: encodeHeaders(resp), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: fr.Priority, + }, decoder) + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + + writeFinalData(reverseFramer, streamId, buf, int(http2Handler.maxFrameSize)) + reverseFramer.WriteGoAway(streamId, http2.ErrCodeRefusedStream, nil) + return STATUS_BLOCKED + } + } else { + response := makeHttpResponse(nil, headerFields) + if http2Handler.verbose { + log.Printf("%d Headers received, %d, flagged: %v", http2Handler.id, response.StatusCode, http2Handler.verbose) + } + http2Handler.rwMutex.Lock() + http2Handler.lastHttpResponse[streamId] = response + http2Handler.lastHttpResponse[streamId].Request = http2Handler.lastHttpRequest[streamId] + http2Handler.rwMutex.Unlock() + + contentType := response.Header.Get("Content-Type") + contentLength, _ := strconv.ParseInt(response.Header.Get("Content-Length"), 0, 64) + if !isContentTypeFilterable(contentType, contentLength) { + writeHeadersImmediately = true + } + } + + header := http2.HeadersFrameParam{ + StreamID: streamId, + BlockFragment: encodeHeaderFields(headerFields), + EndStream: fr.StreamEnded(), + EndHeaders: fr.HeadersEnded(), + PadLength: 0, + Priority: fr.Priority, + } + + http2Handler.rwMutex.Lock() + http2Handler.lastHeadersMap[f.Header().StreamID] = headerFields + http2Handler.lastHeadersBlock[f.Header().StreamID] = &header + http2Handler.rwMutex.Unlock() + + if writeHeadersImmediately { + http2Handler.writeDataAndHeaders(decoder, directFramer, f.Header().StreamID, fr.StreamEnded()) + } + case http2.FramePriority: + fr := f.(*http2.PriorityFrame) + directFramer.WritePriority(f.Header().StreamID, fr.PriorityParam) + case http2.FrameRSTStream: + fr := f.(*http2.RSTStreamFrame) + directFramer.WriteRSTStream(f.Header().StreamID, fr.ErrCode) + case http2.FrameSettings: + fr := f.(*http2.SettingsFrame) + if !client { + http2Handler.connectionReadyForData = true //once server sent the settings we're good to go + } + + if fr.IsAck() { + directFramer.WriteSettingsAck() + } else { + params := make([]http2.Setting, 0) + for i := 0; i < fr.NumSettings(); i++ { + setting := fr.Setting(i) + params = append(params, setting) + if setting.ID == http2.SettingHeaderTableSize && client { + decoder.SetMaxDynamicTableSize(setting.Val) + } + if setting.ID == http2.SettingMaxFrameSize && client { + if http2Handler.maxFrameSize > setting.Val { + http2Handler.maxFrameSize = setting.Val + } + } + + } + directFramer.WriteSettings(params...) + } + + case http2.FramePushPromise: + fr := f.(*http2.PushPromiseFrame) + directFramer.WritePushPromise(http2.PushPromiseParam{ + + StreamID: f.Header().StreamID, + PromiseID: fr.PromiseID, + BlockFragment: fr.HeaderBlockFragment(), + EndHeaders: fr.HeadersEnded(), + PadLength: 0, + }) + case http2.FramePing: + fr := f.(*http2.PingFrame) + directFramer.WritePing(fr.IsAck(), fr.Data) + case http2.FrameGoAway: + fr := f.(*http2.GoAwayFrame) + directFramer.WriteGoAway(fr.LastStreamID, fr.ErrCode, fr.DebugData()) + case http2.FrameWindowUpdate: + fr := f.(*http2.WindowUpdateFrame) + directFramer.WriteWindowUpdate(f.Header().StreamID, fr.Increment) + case http2.FrameContinuation: + fr := f.(*http2.ContinuationFrame) + directFramer.WriteContinuation(f.Header().StreamID, fr.HeadersEnded(), fr.HeaderBlockFragment()) + default: + fr := f.(*http2.UnknownFrame) + directFramer.WriteRawFrame(f.Header().Type, f.Header().Flags, f.Header().StreamID, fr.Payload()) + } + + return STATUS_OK +} + +func isContentTypeFilterable(contentType string, contentLength int64) bool { + if contentLength > MAX_FILTERABLE_LENGTH { + return false + } + if strings.Contains(contentType, "protobuf") { + return false + } + + result := strings.Contains(contentType, "html") || strings.Contains(contentType, "json") + if ENABLE_IMAGE_FILTERING && !result { + result = result || + strings.Contains(contentType, "image/png") || + strings.Contains(contentType, "image/jpg") || + strings.Contains(contentType, "image/jpeg") || + strings.Contains(contentType, "image/webp") + } + return result +} + +func decodeAllHeaders(framer *http2.Framer, fr *http2.HeadersFrame, decoder *hpack.Decoder) ([]hpack.HeaderField, []byte) { + buf := new(bytes.Buffer) + res := make([]hpack.HeaderField, 0) + + hostIndex := 0 + pathIndex := 0 + cookieIndex := 0 + decoder.SetEmitEnabled(true) + decoder.SetMaxStringLength(16 << 20) + decoder.SetEmitFunc(func(hf hpack.HeaderField) { + if len(hf.Name) > 0 { + if hf.Name == ":path" { + pathIndex = len(res) + } else if hf.Name == ":authority" { + hostIndex = len(res) + } else if hf.Name == "cookie" { + cookieIndex = len(res) + } + res = append(res, hf) + } + }) + defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) + defer decoder.Close() + + buf.Write(fr.HeaderBlockFragment()) + _, err := decoder.Write(fr.HeaderBlockFragment()) + if err != nil { + log.Printf("Error decode %v", err) + } + if fr.HeadersEnded() { + if hostIndex > 0 || pathIndex > 0 { + res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + + if cookieIndex > 0 { + res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) + } + } + + return res, buf.Bytes() + } + for { + if f, err := framer.ReadFrame(); err != nil { + break + } else { + continuationFrame := f.(*http2.ContinuationFrame) // guaranteed by checkFrameOrder + buf.Write(continuationFrame.HeaderBlockFragment()) + _, err = decoder.Write(continuationFrame.HeaderBlockFragment()) + if err != nil { + log.Printf("Error decode %v", err) + } + if continuationFrame.HeadersEnded() { + break + } + } + } + + if hostIndex > 0 || pathIndex > 0 { + res[pathIndex].Value = HostPathForceSafeSearch(res[hostIndex].Value, res[pathIndex].Value) + if cookieIndex > 0 { + res[cookieIndex].Value = CookiePatchSafeSearch(res[hostIndex].Value, res[cookieIndex].Value) + } + } + + return res, buf.Bytes() +} + +func (http2Handler *Http2Handler) writeDataAndHeaders(decoder *hpack.Decoder, directFramer *http2.Framer, streamId uint32, streamEnded bool) { + http2Handler.rwMutex.Lock() + headerFields := http2Handler.lastHeadersMap[streamId] + bodyChunks := http2Handler.responseBodyMapChunks[streamId] + lastHeader := http2Handler.lastHeadersBlock[streamId] + http2Handler.rwMutex.Unlock() + + if lastHeader != nil { + header := &http2.HeadersFrameParam{ + StreamID: streamId, + BlockFragment: encodeHeaderFields(headerFields), + EndStream: false, + EndHeaders: true, + PadLength: 0, + Priority: lastHeader.Priority, + } + header.EndStream = bodyChunks == nil && streamEnded + writeHeaders(directFramer, header, decoder) + + http2Handler.rwMutex.Lock() + delete(http2Handler.lastHeadersBlock, streamId) + delete(http2Handler.lastHeadersMap, streamId) + http2Handler.rwMutex.Unlock() + } + + if bodyChunks != nil { + for i, _ := range bodyChunks { + streamEnd := i == len(bodyChunks)-1 && streamEnded + directFramer.WriteData(streamId, streamEnd, bodyChunks[i]) + } + + http2Handler.rwMutex.Lock() + delete(http2Handler.responseBodyMapChunks, streamId) + http2Handler.rwMutex.Unlock() + } +} + +func writeFinalData(framer *http2.Framer, streamId uint32, data *bytes.Buffer, chunkSize int) { + dataToSend := data.Bytes() + for i := 0; i < len(dataToSend); i += chunkSize { + end := i + chunkSize + dataEnded := false + if end >= len(dataToSend) { + end = len(dataToSend) + dataEnded = true + } + + framer.WriteData(streamId, dataEnded, dataToSend[i:end]) + } +} + +func writeHeaders(framer *http2.Framer, param *http2.HeadersFrameParam, decoder *hpack.Decoder) { + dataToSend := param.BlockFragment + chunkSize := 15 * 1024 + for i := 0; i < len(dataToSend); i += chunkSize { + end := i + chunkSize + headesEnd := false + if end >= len(dataToSend) { + end = len(dataToSend) + headesEnd = true + } + + if i == 0 { + /* decoder.SetEmitEnabled(true) + decoder.SetMaxStringLength(16 << 20) + decoder.SetEmitFunc(func(hf hpack.HeaderField) { + if len(hf.Name) > 0 { + log.Printf("Writing header id:%d, %s:%s", param.StreamID, hf.Name, hf.Value) + } + }) + defer decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) + defer decoder.Close() + + decoder.Write(dataToSend[i:end])*/ + + framer.WriteHeaders(http2.HeadersFrameParam{ + StreamID: param.StreamID, + BlockFragment: dataToSend[i:end], + EndStream: headesEnd && param.EndStream, + EndHeaders: headesEnd, + PadLength: 0, + Priority: param.Priority, + }) + } else { + framer.WriteContinuation(param.StreamID, headesEnd, dataToSend[i:end]) + } + } +} + +func makeHttpRequest(body []byte, header []hpack.HeaderField) *http.Request { + req := http.Request{} + req.Proto = "http/2" + req.ProtoMajor = 2 + req.ProtoMinor = 0 + + req.Header = http.Header{} + scheme := "https" + authority := "" + path := "" + for _, v := range header { + if !strings.HasPrefix(v.Name, ":") { + req.Header.Add(v.Name, v.Value) + } else if v.Name == ":scheme" { + scheme = v.Value + } else if v.Name == ":authority" { + authority = v.Value + } else if v.Name == ":path" { + path = v.Value + } else if v.Name == ":method" { + req.Method = v.Value + } + } + + req.RequestURI = scheme + "://" + authority + path + req.URL, _ = url.ParseRequestURI(req.RequestURI) + req.Host = req.URL.Host + + if len(body) == 0 { + req.Body = http.NoBody + } else { + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + } + return &req +} + +func makeHttpResponse(body []byte, header []hpack.HeaderField) *http.Response { + resp := http.Response{} + + resp.Proto = "http/2" + resp.ProtoMajor = 2 + resp.ProtoMinor = 0 + resp.Header = http.Header{} + for _, v := range header { + if !strings.HasPrefix(v.Name, ":") { + resp.Header.Add(v.Name, v.Value) + } else if v.Name == ":status" { + resp.Status = v.Value + resp.StatusCode, _ = strconv.Atoi(v.Value) + } + } + if len(body) == 0 { + resp.Body = http.NoBody + resp.ContentLength = 0 + } else { + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + } + return &resp +} + +func putResponseBody(bodyParts [][]byte, resp *http.Response) { + if len(bodyParts) == 0 { + resp.Body = http.NoBody + } else { + body := make([]byte, 0) + for _, b := range bodyParts { + body = append(body, b...) + } + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + } +} + +func encodeHeaderFields(fields []hpack.HeaderField) []byte { + buf := new(bytes.Buffer) + encoder := hpack.NewEncoder(buf) + encoder.SetMaxDynamicTableSizeLimit(65536) + buf.Reset() + + for i := 0; i < len(fields); i++ { + encoder.WriteField(fields[i]) + } + return buf.Bytes() +} + +func encodeHeaders(resp *http.Response) []byte { + buf := new(bytes.Buffer) + encoder := hpack.NewEncoder(buf) + // encoder.SetMaxDynamicTableSize(65536) + buf.Reset() + + writeHeader(encoder, ":status", strconv.Itoa(resp.StatusCode)) + writeHeader(encoder, "content-length", strconv.FormatInt(resp.ContentLength, 10)) + for k, vv := range resp.Header { + lowKey := strings.ToLower(k) + for _, v := range vv { + writeHeader(encoder, lowKey, v) + } + } + return buf.Bytes() +} + +func writeHeader(encoder *hpack.Encoder, name, value string) { + encoder.WriteField(hpack.HeaderField{Name: name, Value: value}) +} From 8f60a0703ae96399ee703c78f563cdcb06bae00d Mon Sep 17 00:00:00 2001 From: Dmitriy Goraschenko Date: Fri, 17 Oct 2025 13:49:07 +0200 Subject: [PATCH 27/27] token Device identifier category_name The name of the blocked category device_name base64 encoded device name user_id account email trigger base64 encoded blocked text trigger term when this is blocked by a text trigger platform cv4w, cv4m, cv4a, cv4i --- goproxy/adblock.go | 9 ++++++-- goproxy/goproxystarter.go | 46 ++++++++++++++++++++++++++++++--------- goproxy/http2.go | 25 +++++++++++++-------- goproxy/main.go | 4 +++- 4 files changed, 62 insertions(+), 22 deletions(-) diff --git a/goproxy/adblock.go b/goproxy/adblock.go index 42ff99f..cbd997d 100644 --- a/goproxy/adblock.go +++ b/goproxy/adblock.go @@ -117,13 +117,15 @@ func (am *AdBlockMatcher) addMatcher(category string, listType int) { am.lastCategory = categoryMatcher } -func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPolicy bool) string { +func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, textTrigger string, isRelaxedPolicy bool) string { tags := am.defaultBlockPageTags tags["url_text"] = blockedUrl tags["friendly_url_text"] = blockedUrl tags["message"] = "" tags["matching_category"] = category + tags["text_trigger"] = textTrigger + if isRelaxedPolicy { tags["isRelaxedPolicy"] = "1" } else { @@ -131,7 +133,10 @@ func (am *AdBlockMatcher) GetBlockPage(blockedUrl, category string, isRelaxedPol } tags["showUnblockRequestButton"] = "1" - tags["unblockRequest"] = tags["unblockRequestBase"] + "&category_name=" + url.QueryEscape(category) + "&blocked_request=" + base64.StdEncoding.EncodeToString([]byte(blockedUrl)) + tags["unblockRequest"] = tags["unblockRequestBase"] + + "&category_name=" + url.QueryEscape(category) + + "&blocked_request=" + base64.StdEncoding.EncodeToString([]byte(blockedUrl)) + + "&trigger=" + base64.StdEncoding.EncodeToString([]byte(textTrigger)) res, err := am.BlockPageTemplate.Exec(tags) if err != nil { diff --git a/goproxy/goproxystarter.go b/goproxy/goproxystarter.go index 934b020..30569e0 100644 --- a/goproxy/goproxystarter.go +++ b/goproxy/goproxystarter.go @@ -3,12 +3,12 @@ package main /* #include -typedef int (*adBlockCallback)(char* url, char* category); +typedef int (*adBlockCallback)(char* url, char* category, char* trigger); -static inline int FireAdblockCallback(void* ptr, char* url, char* category) +static inline int FireAdblockCallback(void* ptr, char* url, char* category, char* trigger) { adBlockCallback p = (adBlockCallback)ptr; - return p(url, category); + return p(url, category, trigger); } */ import "C" @@ -67,7 +67,7 @@ var handleConnectFunc HttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (* func initGoProxy() { proxy = goproxy.NewProxyHttpServer() - proxy.Verbose = true + proxy.Verbose = false proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Host == "" { @@ -343,23 +343,31 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert if adBlockMatcher != nil { category, matchType, isRelaxedPolicy := adBlockMatcher.TestUrlBlocked(req.URL.String(), req.Host, req.Referer()) if category != nil && matchType == Included { + categoryParts := strings.Split(*category, "/") + baseCategory := category + if len(categoryParts) > 2 { + baseCategory = &categoryParts[len(categoryParts)-2] + } + url := req.URL.String() if adBlockBlacklistCallback != nil { unsafeUrl := C.CString(url) - unsafeCategory := C.CString(*category) - C.FireAdblockCallback(adBlockBlacklistCallback, unsafeUrl, unsafeCategory) + unsafeCategory := C.CString(*baseCategory) + unsafeTrigger := C.CString("") + C.FireAdblockCallback(adBlockBlacklistCallback, unsafeUrl, unsafeCategory, unsafeTrigger) C.free(unsafe.Pointer(unsafeUrl)) C.free(unsafe.Pointer(unsafeCategory)) + C.free(unsafe.Pointer(unsafeTrigger)) } - log.Printf("Page %s blocked by url, category %s", url, *category) + log.Printf("Page %s blocked by url, category %s", url, *baseCategory) if strings.Contains(req.URL.Host, "vimeo") { req.Header.Set("cookie", CookiePatchSafeSearch(req.URL.Host, req.Header.Get("cookie"))) } response := NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, - adBlockMatcher.GetBlockPage(url, *category, isRelaxedPolicy)) + adBlockMatcher.GetBlockPage(url, *baseCategory, "", isRelaxedPolicy)) req.URL.RawPath = HostPathForceSafeSearch(req.URL.Host, req.URL.RawPath) return req, response @@ -435,8 +443,26 @@ func startGoProxyServer(portHttp, portHttps, portConfigurationServer int16, cert category, matches := adBlockMatcher.TestContainsForbiddenPhrases(bytesData) if category != nil { - log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", resp.Request.URL.String(), *category, strings.Join(matches, ", ")) - message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *category, false) + url := resp.Request.URL.String() + triggers := strings.Join(matches, ", ") + categoryParts := strings.Split(*category, "/") + baseCategory := category + if len(categoryParts) > 2 { + baseCategory = &categoryParts[len(categoryParts)-2] + } + + log.Printf("Page %s blocked, category: %s, found forbidden phrases: %s", url, *baseCategory, triggers) + if adBlockBlacklistCallback != nil { + unsafeUrl := C.CString(url) + unsafeCategory := C.CString(*baseCategory) + unsafeTrigger := C.CString(triggers) + C.FireAdblockCallback(adBlockBlacklistCallback, unsafeUrl, unsafeCategory, unsafeTrigger) + C.free(unsafe.Pointer(unsafeUrl)) + C.free(unsafe.Pointer(unsafeCategory)) + C.free(unsafe.Pointer(unsafeTrigger)) + } + + message := adBlockMatcher.GetBlockPage(resp.Request.URL.String(), *baseCategory, triggers, false) blockedResp := NewResponse(resp.Request, goproxy.ContentTypeHtml, http.StatusForbidden, message) return blockedResp } diff --git a/goproxy/http2.go b/goproxy/http2.go index 482a806..fd100b0 100644 --- a/goproxy/http2.go +++ b/goproxy/http2.go @@ -47,10 +47,9 @@ type Http2Handler struct { func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UConn) bool { log.Print("Running http2 handler for " + r.URL.String()) verbose := false - // if strings.Contains(r.URL.String(), "womenshealthmag") { - // log.Printf("Google %s: serveHttp2Filtering", r.URL.String()) - // verbose = true - // } + if strings.Contains(r.URL.String(), "monday.com") { + verbose = true + } http2Handler := &Http2Handler{ maxFrameSize: 1024, @@ -64,7 +63,10 @@ func serveHttp2Filtering(r *http.Request, rawClientTls *tls.Conn, remote *tls.UC connectionReadyForData: false, rwMutex: &sync.RWMutex{}, verbose: verbose, - id: time.Now().Unix(), + id: time.Now().UnixNano() / (1 << 22), + } + if verbose { + log.Printf("Monday %s: serveHttp2Filtering - %d", r.URL.String(), http2Handler.id) } go func() { http2Handler.processHttp2Stream(rawClientTls, remote) @@ -160,15 +162,17 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F bodyChunks = http2Handler.responseBodyMapChunks[streamId] http2Handler.rwMutex.RUnlock() if http2Handler.verbose && fr.StreamEnded() { - log.Printf("%d Stream ended, flagged: %v", http2Handler.id, client, http2Handler.verbose) + log.Printf("%d Stream ended, client %v, flagged: %v", http2Handler.id, client, http2Handler.verbose) } streamEnded := fr.StreamEnded() || force if !whitelisted && lastHttpResponse != nil && !client { contentType := lastHttpResponse.Header.Get("Content-Type") contentLength := lastHttpResponse.ContentLength - isContentTypeFilterable := isContentTypeFilterable(contentType, contentLength) + isContentTypeFilterable := isContentTypeFilterable(contentType, contentLength) && lastHttpResponse.StatusCode < 300 putResponseBody(bodyChunks, lastHttpResponse) - + if http2Handler.verbose { + log.Printf("%d Stream DATA, content Filterable: %v", http2Handler.id, isContentTypeFilterable) + } if isContentTypeFilterable && streamEnded { if contentLength > MIN_FILTERABLE_LENGTH { http2Handler.rwMutex.RLock() @@ -284,8 +288,11 @@ func (http2Handler *Http2Handler) readFrame(directFramer, reverseFramer *http2.F contentType := response.Header.Get("Content-Type") contentLength, _ := strconv.ParseInt(response.Header.Get("Content-Length"), 0, 64) - if !isContentTypeFilterable(contentType, contentLength) { + if !isContentTypeFilterable(contentType, contentLength) || response.StatusCode > 300 { writeHeadersImmediately = true + if http2Handler.verbose { + log.Printf("%d Headers writing immediately", http2Handler.id) + } } } diff --git a/goproxy/main.go b/goproxy/main.go index 7ecff07..158a1c5 100644 --- a/goproxy/main.go +++ b/goproxy/main.go @@ -26,7 +26,6 @@ var certsException = make(map[string]bool) var logFilePath = "" var logFileHandle *os.File var isImageFilteringEnabled = false - var mainInteropSyncMutex sync.Mutex //export AddCertException @@ -208,5 +207,8 @@ func main() { func test() { log.Printf("main: starting HTTP server") + AdBlockMatcherInitialize() + AdBlockMatcherParseRuleFile(C.CString("./rules.txt"), C.CString(""), 1) + AdBlockMatcherBuild() startGoProxyServer(14600, 14501, 14502, "rootCertificate.pem", "rootPrivateKey.pem") }