From 3cfbb533b6c67647df24983523003d81f86b0320 Mon Sep 17 00:00:00 2001 From: Riley Apeldoorn Date: Wed, 20 Dec 2023 11:35:33 +0100 Subject: [PATCH 1/5] Types: unreserve more characters --- Network/HTTP/Types/URI.hs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Network/HTTP/Types/URI.hs b/Network/HTTP/Types/URI.hs index 0770723..90f455d 100644 --- a/Network/HTTP/Types/URI.hs +++ b/Network/HTTP/Types/URI.hs @@ -275,12 +275,11 @@ ord8 :: Char -> Word8 ord8 = fromIntegral . ord unreservedQS, unreservedPI :: [Word8] -unreservedQS = map ord8 "-_.~" --- FIXME: According to RFC 3986, the following are also allowed in path segments: --- "!'()*;" --- --- https://www.rfc-editor.org/rfc/rfc3986#section-3.3 -unreservedPI = map ord8 "-_.~:@&=+$," +-- NOTE: semicolons are also allowed in query strings, but putting them in the unreserved +-- section here breaks the roundtrip tests due to the inclusion of the semicolon in the +-- `queryStringSeparators` above +unreservedQS = map ord8 "-_.~:@$,()'!*" +unreservedPI = map ord8 "-_.~:@&=+$,()'!*;" -- | Percent-encoding for URLs. -- From 0239ee25538402192c3ff8ea3035b881c3b66489 Mon Sep 17 00:00:00 2001 From: Riley Apeldoorn Date: Wed, 27 Dec 2023 11:32:29 +0100 Subject: [PATCH 2/5] Types: un-urlencode `:` in renderQueryPartialEscape doctest --- Network/HTTP/Types/URI.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Network/HTTP/Types/URI.hs b/Network/HTTP/Types/URI.hs index 90f455d..d6b5bb8 100644 --- a/Network/HTTP/Types/URI.hs +++ b/Network/HTTP/Types/URI.hs @@ -527,7 +527,7 @@ type PartialEscapeQuery = [PartialEscapeQueryItem] -- If you want a question mark (@?@) added to the front of the result, use 'True'. -- -- >>> renderQueryPartialEscape True [("a", [QN "x:z + ", QE (encodeUtf8 "They said: \"שלום\"")])] --- "?a=x:z + They%20said%3A%20%22%D7%A9%D7%9C%D7%95%D7%9D%22" +-- "?a=x:z + They%20said:%20%22%D7%A9%D7%9C%D7%95%D7%9D%22" -- -- @since 0.12.1 renderQueryPartialEscape :: Bool -> PartialEscapeQuery -> B.ByteString From fc98d54b7bbc8a6b2f18f0131a79dce5febdc05a Mon Sep 17 00:00:00 2001 From: Riley Apeldoorn Date: Wed, 31 Jan 2024 15:28:02 +0100 Subject: [PATCH 3/5] Tests: add property tests for url encoding/decoding roundtrips --- test/Network/HTTP/Types/URISpec.hs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/Network/HTTP/Types/URISpec.hs b/test/Network/HTTP/Types/URISpec.hs index 1c75493..28e9ae8 100644 --- a/test/Network/HTTP/Types/URISpec.hs +++ b/test/Network/HTTP/Types/URISpec.hs @@ -85,6 +85,14 @@ spec = do it "still encodes the same (query)" $ mkGoldenFile "urlEncode-query" $ urlEncode True asciis + it "roundtrips (urlDecode True . urlEncode True)" $ + property (\input -> input == (urlDecode True . urlEncode True) input) + it "roundtrips (urlDecode False . urlEncode False)" $ + property (\input -> input == (urlDecode False . urlEncode False) input) + it "roundtrips (urlDecode False . urlEncode True)" $ + property (\input -> input == (urlDecode False . urlEncode True) input) + it "roundtrips (urlDecode True . urlEncode False)" $ + property (\input -> input == (urlDecode True . urlEncode False) input) describe "decodePathSegments" $ do it "is inverse to encodePathSegments" $ @@ -262,4 +270,4 @@ toStrictBS :: BB.Builder -> B.ByteString toStrictBS = BL.toStrict . BB.toLazyByteString toSimpleQuery :: Query -> SimpleQuery -toSimpleQuery q = fmap (fromMaybe "") <$> q +toSimpleQuery q = fmap (fromMaybe "") <$> q From 31db21a0822e8286543718821341bdb2eb07d23b Mon Sep 17 00:00:00 2001 From: Riley Apeldoorn Date: Wed, 31 Jan 2024 17:22:05 +0100 Subject: [PATCH 4/5] Tests: update golden test files --- test/.golden/urlEncode-path/golden | 2 +- test/.golden/urlEncode-query/golden | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/.golden/urlEncode-path/golden b/test/.golden/urlEncode-path/golden index 5900b15..eb6eb7c 100644 --- a/test/.golden/urlEncode-path/golden +++ b/test/.golden/urlEncode-path/golden @@ -1 +1 @@ -%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20%21%22%23$%25&%27%28%29%2A+,-.%2F0123456789:%3B%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file +%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25&'()*+,-.%2F0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file diff --git a/test/.golden/urlEncode-query/golden b/test/.golden/urlEncode-query/golden index b725385..ac4e032 100644 --- a/test/.golden/urlEncode-query/golden +++ b/test/.golden/urlEncode-query/golden @@ -1 +1 @@ -%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20%21%22%23%24%25%26%27%28%29%2A%2B%2C-.%2F0123456789%3A%3B%3C%3D%3E%3F%40ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file +%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25%26'()*%2B,-.%2F0123456789:%3B%3C%3D%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file From 1d461456841915cd3279bcafe333889ad6a0a649 Mon Sep 17 00:00:00 2001 From: Riley Apeldoorn Date: Wed, 31 Jan 2024 17:57:05 +0100 Subject: [PATCH 5/5] Types: add `?` and `/` to unreserved query string characters --- Network/HTTP/Types/URI.hs | 2 +- test/.golden/urlEncode-query/golden | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Network/HTTP/Types/URI.hs b/Network/HTTP/Types/URI.hs index d6b5bb8..824b4b7 100644 --- a/Network/HTTP/Types/URI.hs +++ b/Network/HTTP/Types/URI.hs @@ -278,7 +278,7 @@ unreservedQS, unreservedPI :: [Word8] -- NOTE: semicolons are also allowed in query strings, but putting them in the unreserved -- section here breaks the roundtrip tests due to the inclusion of the semicolon in the -- `queryStringSeparators` above -unreservedQS = map ord8 "-_.~:@$,()'!*" +unreservedQS = map ord8 "-_.~:@$,()'!*/?" unreservedPI = map ord8 "-_.~:@&=+$,()'!*;" -- | Percent-encoding for URLs. diff --git a/test/.golden/urlEncode-query/golden b/test/.golden/urlEncode-query/golden index ac4e032..172e199 100644 --- a/test/.golden/urlEncode-query/golden +++ b/test/.golden/urlEncode-query/golden @@ -1 +1 @@ -%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25%26'()*%2B,-.%2F0123456789:%3B%3C%3D%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file +%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25%26'()*%2B,-./0123456789:%3B%3C%3D%3E?@ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF \ No newline at end of file