public static void testHandler分批处理集合() {
    //业务数据
    List<String>  dataList=new ArrayList<>();
    for (int i = 0; i < 100000; i++) {
      dataList.add(i+"");
    }
    int total = dataList.size();
    // 每次分批处理个数
    int num=500;
    long startTime = System.currentTimeMillis();
    List<String> result=new ArrayList<>();
    // 数据量大于500 进行分批,小于等于则不用分批执行
    if (total > 500) {
      try {
        result = Stream.iterate(0, i -> i + 1).limit(dataList.size() / num + 1).parallel().map(i -> {
          //获取每一批的数据
          List<String> perList = dataList.parallelStream().skip(i * num)
              .limit(num)
              .collect(Collectors.toList());
          //业务处理
          return  queryThisData(perList,i);
        }).filter(o -> Objects.nonNull(o))
            .flatMap(Collection::parallelStream)
            .collect(Collectors.toList());
      } catch (Exception e) {
        e.printStackTrace();
      }
    } else {
      result = queryThisData(dataList, total);
    }


    long endTime = System.currentTimeMillis();
    System.out.println("并发批量处理耗时:"+(endTime-startTime)+" 数据总数:"+result.size());

    //不并发分批
    List<String> result2=new ArrayList<>();
    List<List<String>> parList=new ArrayList<>();
    int size = dataList.size() % num == 0 ? (dataList.size() / num) : (dataList.size() / num + 1);
    for (int i = 0; i < size ; i++) {
      List<String> list=new ArrayList<>();
      for (int j=i*num;j<num*(i+1);j++){
        list.add(dataList.get(j));
      }
      parList.add(list);
    }
    for (int i = 0; i < parList.size(); i++) {
      result2.addAll(queryThisData(parList.get(i),i));
    }
    System.out.println("不并发批量处理耗时:"+(System.currentTimeMillis()-endTime)+" 数据总数:"+result2.size());
  }

  private static List<String> queryThisData(List<String> perList,int i) {
    try {
      Thread.sleep(100);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
    return perList;
  }

并发批量处理耗时:3190 数据总数:100000
不并发批量处理耗时:21892 数据总数:100000
Logo

北京人形旗下天工造物具身智能开源社区,聚焦具身天工与慧思开物两大平台

更多推荐