We have a crawler running on our 2GB, 2 CPU ubuntu machine, for the past few months we have noticed a large number of twisted.internet.error.TimeoutError in the crawler stats, which is causing slowness in our entire crawl process. Also, we noticed that this is happening only when we crawling sites with large contents In it and crawler is using only 50% (1 core ) of the CPU in the system.

I tried changing the DOWNLOAD_DELAY settings and I couldn't make any progress in that.  Please let me know if you have a better way to solve this problem.

Attaching a sample stats from the crawler

{
  "memusage/startup": 337682432,
  "downloader/exception_type_count/twisted.internet.error.TimeoutError": 58457,
  "splash/execute/response_count/500": 1507,
  "downloader/response_status_count/500": 1507,
  "downloader/request_count": 1081283,
  "memusage/warning_notified": 1,
  "finish_time": "2020-07-10 06:38:44.584240",
  "item_scraped_count": 11468349,
  "downloader/response_bytes": 31299223551,
  "error_item_count": 2,
  "retry/reason_count/500 Internal Server Error": 1454,
  "memusage/warning_reached": 1,
  "splash/execute/response_count/504": 17,
  "splash/execute/response_count/502": 379,
  "scheduler/enqueued": 2102353,
  "splash/execute/response_count/400": 29,
  "log_count/WARNING": 1794,
  "memusage/max": 1925443584,
  "splash/execute/response_count/200": 1020130,
  "siteclarity_item_count": 1021043,
  "downloader/response_status_count/502": 379,
  "downloader/exception_type_count/twisted.web._newclient.ResponseNeverReceived": 60,
  "retry/reason_count/twisted.web._newclient.ResponseNeverReceived": 59,
  "start_time": "2020-06-19 19:56:17.613957",
  "downloader/response_status_count/504": 17,
  "ignore/unhandled_exception": 1,
  "retry/max_reached": 472,
  "memusage/limit_notified": 1,
  "downloader/exception_count": 58517,
  "type/canonical": 1136,
  "downloader/response_status_count/302": 49,
  "type/doc": 1019907,
  "downloader/response_status_count/503": 265,
  "dupefilter/filtered": 13580051,
  "retry/reason_count/504 Gateway Time-out": 17,
  "log_count/INFO": 1063157,
  "splash/execute/response_count/301": 327,
  "robots_txt/recording/blocked/doc": 55911322,
  "retry/reason_count/twisted.internet.error.TimeoutError": 58092,
  "scheduler/enqueued/disk": 2102353,
  "downloader/response_status_count/404": 61,
  "splash/execute/response_count/302": 49,
  "splash/execute/response_count/404": 61,
  "splash/execute/request_count": 1021070,
  "retry/reason_count/503 Service Unavailable": 256,
  "response_received_count": 1020619,
  "downloader/response_status_count/307": 2,
  "downloader/request_method_count/POST": 1081283,
  "memusage/limit_reached": 1,
  "downloader/response_count": 1022766,
  "scheduler/dequeued": 2102353,
  "downloader/request_bytes": 1231343961,
  "scheduler/dequeued/disk": 2102353,
  "downloader/response_status_count/301": 327,
  "finish_reason": "finished",
  "retry/count": 60213,
  "request_depth_max": 4,
  "splash/execute/response_count/503": 265,
  "downloader/response_status_count/400": 29,
  "splash/execute/response_count/307": 2,
  "log_count/ERROR": 1990,
  "retry/reason_count/502 Bad Gateway": 335,
  "downloader/response_status_count/200": 1020130
}

(edited for code formatting)