运行环境

oracle 圣何塞 debian arm 4C 24G

JDK 21, 不能在本地上复现 代理程序在 服务器上

htop 观察到 是某一个核心突然满载 然后就卡住了

代码

public class Main { static Log log = Log.get(); static final Gson gson = new Gson(); static List<Image> images = null; static int maxThreads = 100; // 控制固定线程池的大小 static ExecutorService virtualThreadPool = Executors.newFixedThreadPool(maxThreads, Thread.ofVirtual().factory()); static String filePath = "/root/java_work/imglist/"; // 失败 list static List<String> failList = Lists.newArrayList(); public static void main(String[] args) { OkHttpUtils.setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 6789))); List<String> types = List.of(".jpg", ".png"); try { images = gson.fromJson(new FileReader("output-2024-4-16.json"), new TypeToken<List<Image>>(){}.getType()); } catch (IOException e) { log.info("读取文件失败: {}", e.getMessage()); return; } log.info("加载到的图片数量: {}", images.size()); images.forEach(image -> virtualThreadPool.submit(() -> { for (String type : types) { String url = convertPreviewToImageUrl(image.getHref(), type); if (attemptToDownloadImage(url,0)) { image.setSourceUrl(url); // 更新 Image 对象 break; } } })); virtualThreadPool.shutdown(); while (!virtualThreadPool.isTerminated()) { Thread.onSpinWait(); } // 将更新后的 images 列表写回到 JSON 文件 writeImagesToJson(images, "output-2024-4-16-ok.json"); writeImagesToJson(failList, "output-2024-4-16-fail.json"); } private static boolean attemptToDownloadImage(String url, int retryCount) { if (retryCount >= 3) { log.info("重试次数过多，放弃下载: {}", url); failList.add(url); return false; } try (Response respOnse= OkHttpUtils.get(url, Headers.of("Connection", "close"))) { switch (response.code()) { case 200: log.info("成功下载图片: {}", url); byte[] bytes = Objects.requireNonNull(response.body()).bytes(); writeImageToFile(bytes, url.substring(url.lastIndexOf('/') + 1)); return true; case 404: log.info("图片不存在: {}", url); return false; case 429: log.info("请求过于频繁，需要稍后重试: {}", url); handleRateLimiting(); return attemptToDownloadImage(url, retryCount + 1); default: log.info("其他 HTTP 响应: {}", response.code()); return false; } } catch (Exception e) { log.error("请求图片时出错: {}", e.fillInStackTrace()); handleRateLimiting(); return attemptToDownloadImage(url, retryCount + 1); } } private static void handleRateLimiting() { try { log.info("等待 5 秒后重试"); Thread.sleep(5000); // 延迟 5 秒后重试 } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } public static String convertPreviewToImageUrl(String previewUrl, String type) { String id = previewUrl.substring(previewUrl.lastIndexOf('/') + 1); return String.format("https://w.wallhaven.cc/full/%s/wallhaven-%s%s", id.substring(0, 2), id, type); } public static void writeImagesToJson(List<?> objects,String fileName) { try (FileWriter writer = new FileWriter(fileName)) { gson.toJson(objects, writer); } catch (IOException e) { log.error("写入文件时出错: {}", e.getMessage()); } } public static void writeImageToFile(byte[] bytes, String fileName) { try (FileOutputStream fos = new FileOutputStream(filePath+fileName)) { fos.write(bytes); log.info("成功写入文件: {}", fileName); } catch (IOException e) { log.error("写入文件时出错: {}", e.getMessage()); } } } 

https://gist.github.com/dnslin/fe657f9df08f4286c197c5e9e5fd6a51