{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":174680350,"defaultBranch":"master","name":"crawly","ownerLogin":"elixir-crawly","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2019-03-09T10:37:03.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/82837769?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1712670606.0","currentOid":""},"activityList":{"items":[{"before":"2a6908f1d4ea82f673af4a5788f564f5cc9ce509","after":"711dba0a0aa33a915609b20e6d08b074d0c7c52f","ref":"refs/heads/master","pushedAt":"2024-04-29T19:45:40.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Take the RobotsTxt User-Agent from the Request (#294)\n\n* Get the correct user agent from the request\r\n\r\n* Add debug logging to find out if it works as expected\r\n\r\n* Unpack the user-agent correctly\r\n\r\n* Use the enhanced gollum fork\r\n\r\n* Handle a missing user-agent header\r\n\r\n* Upgrade httppoison to 2.2\r\n\r\n* Add a helper function to get headers\r\n\r\n* Add a test for the user-agent\r\n\r\n---------\r\n\r\nCo-authored-by: Andreas Donig <git@innwiese.de>","shortMessageHtmlLink":"Take the RobotsTxt User-Agent from the Request (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2237106627\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/294\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/294/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/294\">#294</a>)"}},{"before":"bc245e085cb817f52d2bcfb41005a9b8a5bdea6f","after":"2a6908f1d4ea82f673af4a5788f564f5cc9ce509","ref":"refs/heads/master","pushedAt":"2024-04-24T13:17:17.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"[Fix] Resolve Protocol.UndefinedError by using fallback (#287)","shortMessageHtmlLink":"[Fix] Resolve Protocol.UndefinedError by using fallback (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2155000923\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/287\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/287/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/287\">#287</a>)"}},{"before":"b82a3f10cedd9ea084c1ee22c7e5e7001e917f07","after":"bc245e085cb817f52d2bcfb41005a9b8a5bdea6f","ref":"refs/heads/master","pushedAt":"2024-04-09T13:47:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Update version to 0.17.0","shortMessageHtmlLink":"Update version to 0.17.0"}},{"before":"6351bae76d21680bec78f210e370b2e9ca675a5e","after":"b82a3f10cedd9ea084c1ee22c7e5e7001e917f07","ref":"refs/heads/master","pushedAt":"2024-04-09T11:37:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Update README.md\n\nRemove coveralls badge","shortMessageHtmlLink":"Update README.md"}},{"before":"34e7672832e1b5715353cf2345430a3959c513d5","after":"6351bae76d21680bec78f210e370b2e9ca675a5e","ref":"refs/heads/master","pushedAt":"2024-04-09T10:41:09.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Update HTTPoison to 2.2 (#293)","shortMessageHtmlLink":"Update HTTPoison to 2.2 (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2233136950\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/293\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/293/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/293\">#293</a>)"}},{"before":null,"after":"6d939f8beec1c17f499fca0480f8e4438219cd59","ref":"refs/heads/update_httpoison","pushedAt":"2024-04-09T10:36:37.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Update HTTPoison to 2.2","shortMessageHtmlLink":"Update HTTPoison to 2.2"}},{"before":"6c1316170d2ef521c2665e59854c7964c066b92a","after":null,"ref":"refs/heads/278_fix_autostart","pushedAt":"2024-04-09T10:30:24.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"}},{"before":"afc6915e81f2d98bfd7ec576f20a640ce6f0d730","after":"34e7672832e1b5715353cf2345430a3959c513d5","ref":"refs/heads/master","pushedAt":"2024-04-09T10:30:21.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Automatically start HTTP api when crawly starts (#292)","shortMessageHtmlLink":"Automatically start HTTP api when crawly starts (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2233123062\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/292\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/292/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/292\">#292</a>)"}},{"before":null,"after":"6c1316170d2ef521c2665e59854c7964c066b92a","ref":"refs/heads/278_fix_autostart","pushedAt":"2024-04-09T10:28:24.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Automatically start HTTP api when crawly starts","shortMessageHtmlLink":"Automatically start HTTP api when crawly starts"}},{"before":"385c54e24f384a8c76fa09a3d51005daa91b74d8","after":null,"ref":"refs/heads/patch-2","pushedAt":"2024-04-09T10:24:12.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"}},{"before":"d31375a57f93bd708d095345440253d0f5a42a00","after":"afc6915e81f2d98bfd7ec576f20a640ce6f0d730","ref":"refs/heads/master","pushedAt":"2024-04-09T10:24:09.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Use Crawly.Engine.list_know_spiders instead of Crowly.running_spiders for API (#291)\n\nWhat changed:\r\n\r\n- use Crawly.Engine.list_know_spiders insted of Crowly.running_spiders\r\n- fix spider_name value path\r\n\r\nCo-authored-by: Constantin Angheloiu <constantin.angheloiu@gmail.com>","shortMessageHtmlLink":"Use Crawly.Engine.list_know_spiders instead of Crowly.running_spiders…"}},{"before":null,"after":"385c54e24f384a8c76fa09a3d51005daa91b74d8","ref":"refs/heads/patch-2","pushedAt":"2024-04-09T10:22:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Use Crawly.Engine.list_know_spiders instead of Crowly.running_spiders for API\n\nWhat changed:\n\n- use Crawly.Engine.list_know_spiders insted of Crowly.running_spiders\n- fix spider_name value path","shortMessageHtmlLink":"Use Crawly.Engine.list_know_spiders instead of Crowly.running_spiders…"}},{"before":"cdc3727e63eaa0c18af3f4472da8bb25426194d8","after":"d31375a57f93bd708d095345440253d0f5a42a00","ref":"refs/heads/master","pushedAt":"2024-04-09T09:56:03.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Update returned value to avoid VSCode ElixirLS warning (#284)\n\nNew to Elixir and decided to try using Crawly as a first app. I was getting a warning saying that I was returning the wrong type when copying the sample crawler in the README. Finally figured out it was because I was just returning `%{}` instead of `%Crawly.ParsedItem{}` as the former was what was shown in the README. This is really minor as it didn't affect the correctness of the program, but I figured it might be helpful to have the README so that it didn't trigger this warning.\r\n\r\nFor reference, here's the error text:\r\n```\r\nType mismatch for @callback parse_item/1 in Crawly.Spider behaviour.\r\n\r\nExpected type:\r\n%Crawly.ParsedItem{\r\n  :items => [map()],\r\n  :requests => [\r\n    %Crawly.Request{\r\n      :headers => [{binary(), binary()}],\r\n      :middlewares => [atom()],\r\n      :options => Keyword.t(binary()),\r\n      :prev_response => %{},\r\n      :retries => non_neg_integer(),\r\n      :url => binary()\r\n    }\r\n  ]\r\n}\r\n\r\nActual type:\r\n%{:items => [%{:title => binary()}, ...], :requests => []}\r\nElixirLS Dialyzer\r\n```","shortMessageHtmlLink":"Update returned value to avoid VSCode ElixirLS warning (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2092727554\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/284\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/284/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/284\">#284</a>)"}},{"before":"ff27c12eeed99036c17737fb2055d0f27dec1f2d","after":null,"ref":"refs/heads/render_server_integration","pushedAt":"2024-04-09T09:52:09.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"}},{"before":"bfae1b533b9fc9ff2f401dcd72825e1fae65fb91","after":"cdc3727e63eaa0c18af3f4472da8bb25426194d8","ref":"refs/heads/master","pushedAt":"2024-04-09T09:52:05.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"A new fetcher for Puppeteer based JS rendering (#272)\n\nCreates a splash replacement. I have tested it with just one target so far,\r\nit's hard to say it's perfect, but it might be an alternative to splash\r\nthat is seems to be a bit dead right now","shortMessageHtmlLink":"A new fetcher for Puppeteer based JS rendering (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1863133646\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/272\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/272/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/272\">#272</a>)"}},{"before":"3ad0f1f7a018782dc51fb1002d8fe32b32cb4a80","after":"ff27c12eeed99036c17737fb2055d0f27dec1f2d","ref":"refs/heads/render_server_integration","pushedAt":"2024-04-09T09:50:12.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"A new fetcher for Puppeteer based JS rendering\n\nCreates a splash replacement. I have tested it with just one target so far,\nit's hard to say it's perfect, but it might be an alternative to splash\nthat is seems to be a bit dead right now","shortMessageHtmlLink":"A new fetcher for Puppeteer based JS rendering"}},{"before":"f863b5b278b9c5e78f10c3a3d576b772a7397455","after":"bfae1b533b9fc9ff2f401dcd72825e1fae65fb91","ref":"refs/heads/master","pushedAt":"2024-04-09T09:40:45.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Switch Crawly to use GH actions (#289)\n\nI decided to use Gh actions instead of circle/coveralls,\r\nas I am getting quite a bit of problems when PRs are not\r\nadequately built, or tests fail because overalls do not\r\naccept the new report.\r\n\r\nThe new version is quite simplistic.\r\nIt does not depend on external tools (just github). And I hope builds will be more stable.","shortMessageHtmlLink":"Switch Crawly to use GH actions (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2222798299\" data-permission-text=\"Title is private\" data-url=\"https://github.com/elixir-crawly/crawly/issues/289\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/elixir-crawly/crawly/pull/289/hovercard\" href=\"https://github.com/elixir-crawly/crawly/pull/289\">#289</a>)"}},{"before":"01b05de43c8bd56df6204f3f5f570197e9aa9016","after":"5139ceb18dd2244994df5a36e8ac687a18c0992b","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T09:25:51.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Switch Crawly to use GH actions\n\nI decided to use Gh actions instead of circle/coveralls,\nas I am getting quite a bit of problems when PRs are not\nadequately built, or tests fail because overalls do not\naccept the new report.\n\nThe new version is quite simplistic.\nIt does not depend on external tools (just github). And I hope builds will be more stable.","shortMessageHtmlLink":"Switch Crawly to use GH actions"}},{"before":"9da53fd03709255dcc71748ce8c4122607877192","after":"01b05de43c8bd56df6204f3f5f570197e9aa9016","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T09:20:15.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Add github actions coverage report","shortMessageHtmlLink":"Add github actions coverage report"}},{"before":"eae306078983c3dc12d216d60eccf2ee1ae6b78d","after":"9da53fd03709255dcc71748ce8c4122607877192","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T09:09:33.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Add github actions coverage report","shortMessageHtmlLink":"Add github actions coverage report"}},{"before":"73ad86bc424215056c644aa2435753817e3bad25","after":"eae306078983c3dc12d216d60eccf2ee1ae6b78d","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T08:55:48.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Add github actions coverage report","shortMessageHtmlLink":"Add github actions coverage report"}},{"before":"a375dbb557c9015eac587d18a764f6acaa7602cb","after":"73ad86bc424215056c644aa2435753817e3bad25","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T08:49:15.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Add github actions coverage report","shortMessageHtmlLink":"Add github actions coverage report"}},{"before":"8ed5cd2f5b3288c361e3ef5c725ab73416173838","after":"a375dbb557c9015eac587d18a764f6acaa7602cb","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T08:43:42.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"6318fc9624c61b043fcf0c6c34db92d48c4dfb91","after":"8ed5cd2f5b3288c361e3ef5c725ab73416173838","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T08:37:54.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"f3e0f185ce56d8485acea4698b4333a472bf9b4f","after":"6318fc9624c61b043fcf0c6c34db92d48c4dfb91","ref":"refs/heads/gh_actions","pushedAt":"2024-04-09T08:33:54.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"1e94644355160cb8d06a5f563f15b587fd20fb40","after":"f3e0f185ce56d8485acea4698b4333a472bf9b4f","ref":"refs/heads/gh_actions","pushedAt":"2024-04-03T12:55:25.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"d3f4d8f1f7847347f4d49f34eb84c64305584157","after":"1e94644355160cb8d06a5f563f15b587fd20fb40","ref":"refs/heads/gh_actions","pushedAt":"2024-04-03T12:51:31.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"36207ad0f08a7e1f69c0ffde61d5ff68a708483b","after":"d3f4d8f1f7847347f4d49f34eb84c64305584157","ref":"refs/heads/gh_actions","pushedAt":"2024-04-03T12:44:46.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"1bd1cd6c39b72441503849cfc5e778d6392d37ab","after":"36207ad0f08a7e1f69c0ffde61d5ff68a708483b","ref":"refs/heads/gh_actions","pushedAt":"2024-04-03T12:27:57.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"remove circle","shortMessageHtmlLink":"remove circle"}},{"before":"443b379e117067a7852e59962a32bf8f5c31585f","after":"1bd1cd6c39b72441503849cfc5e778d6392d37ab","ref":"refs/heads/gh_actions","pushedAt":"2024-04-03T12:22:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"oltarasenko","name":null,"path":"/oltarasenko","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48589?s=80&v=4"},"commit":{"message":"Try GH actions","shortMessageHtmlLink":"Try GH actions"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEPVU7VQA","startCursor":null,"endCursor":null}},"title":"Activity · elixir-crawly/crawly"}