devbox
笔触狂放9
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

java VS Excel公式 VS 影刀RPA 之 python的 提速 方法_影刀引用python基础

作者：笔触狂放9 | 2024-03-21 16:43:56

赞

踩

影刀引用python基础

基于我目前开发了几个影刀RPA的感受及 python性能提升的研究，及对比同样功能java实现

感觉提升RPA途径：

1、可以使用Excel的公式，这个效率还行，能充分利用计算机的CPU的线程、进程。

2、java或c#开发的应用程序， java稍微麻烦一点要安装java JDK 并且要配置号路径，设定一个批处理文件即可，

3、影刀的for 或 python的其他循环的方法，也效果不好，我测试是要 5个小时的，用java实现 90秒即可实现。速度是python常规方法的200倍，是使用python加速方法（apply()）的128倍，我测试时数据是这样子的，网上大咖说python某些场景下，比其他开发语言慢200-300倍，是一致的，毕竟他是解释性语言，不是编译的，每一次都要解释之后才能运行。你有多少此循环，就有多少次解释，然后再执行。

说说我 python 提速的途径：，

a、我试过taichi，不支持字符串，仅支持纯数值

b、pypy3安装有问题，没安装上，

c、其他的加速工具，要么重新定义新的数据类型，要么改写之前写好的python代码。

附上我python 测试的几段代码，写得比较乱，但是主要方法发都有的：

  1. # import pandas as pd
  2. # import numpy as np
  3. import pandas as pd  # modin.
  4. import time  # 引入time模块
  5. # import taichi as ti
  6.  
  7. # ti.init()
  8. # from distributed import Client
  9.  
  10. # client = Client()
  11. # df = pd.DataFrame({'month': [1, 4, 7, 10],
  12. #                    'year': [2012, 2014, 2013, 2014],
  13. #                    'sale': [55, 40, 84, 31]})
  14. # print(df)
  15. # df.set_index('year')  #如果索引列有数据重复也没关系
  16. # print(df)
  17. # aa = np.random.randn(6)
  18. # print('aa:{0}',aa)
  19. # --------------------------------------
  20. # df = pd.DataFrame({'a': np.random.randn(6),
  21. #                    'b': ['foo', 'bar'] * 3,
  22. #                    'c': np.random.randn(6)})
  23. # print('----------------')
  24. # def my_test(a, b):
  25. #     return a + b
  26. #
  27. # print(df)
  28. # df['Value'] = df.apply(lambda row: my_test(row['a'], row['c']), axis=1,result_type=None)  # 方法1
  29. # print(df)
  30. #
  31. # df['Value2'] = df['a'] + df['c']  # 方法2
  32. # print(df)
  33. # ------------------------------------
  34. dfListFromDb = pd.read_csv('D:\listFromDb.csv',header=None,usecols=[0,1,2,3])  # ,index_col=0 ,names=['OrderId','BelongTime','ShopCode','ShopCname'] ,index_col=0
  35. # print('dfListFromDb:'+str(dfListFromDb))
  36. # dfListFromDb.reindex(level=None, drop=False, inpalce=False, col_level=0, col_fill=' ')
  37. # dfListFromDb.reindex(range(len(dfListFromDb)))
  38. # dfListFromDb.reset_index(inplace=True,drop=True)
  39. # print('dfListFromDb:' + str(dfListFromDb.index.names))
  40. # print(dfListFromDb.to_string())
  41.  
  42. dfexcelData2Abstract = pd.read_csv('D:\excelData2Abstract.csv',header=None,skiprows=[0],usecols=[0,1])  # ,index_col=1 ,index_col=0
  43. # dfexcelData2Abstract.reset_index(level=None, drop=False, inpalce=False, col_level=0, col_fill=' ')
  44. # dfexcelData2Abstract.reset_index(inplace=True,drop=True)
  45. # dfexcelData2Abstract.reindex(range(len(dfexcelData2Abstract)))
  46.  
  47. # print('dfListFromDb:' + str(dfexcelData2Abstract.index.names))  #
  48. # print('dfexcelData2Abstract.to_string():'+str(dfexcelData2Abstract.to_string()))
  49. # print(dfexcelData2Abstract.to_string())
  50. # print('dfexcelData2Abstract:'+str(dfexcelData2Abstract))
  51. print('xxxxxxxxxxx')
  52. dfexcelData2Abstract2 = dfexcelData2Abstract.copy(True)
  53.  
  54. # @ti.func
  55. def ForloopDoubleLIstRunTime(dfListFromDb, dfexcelData2Abstract2):
  56.     # print('dfexcelData2Abstract2:', dfexcelData2Abstract)
  57.     # dfexcelData2Abstract2.iloc[:, 1:] = np.nan
  58.     # print('aaaaaa')
  59.     # print('After np.nan:dfexcelData2Abstract2:' + str(dfexcelData2Abstract2))  # range
  60.     ticks = time.time()
  61.     for x in range(len(dfexcelData2Abstract.iloc[:, 0])):  # dfexcelData2Abstract2.iloc[:,0]:
  62.         # print(str(x))
  63.         for y in range(len(dfListFromDb.iloc[:, 0])):
  64.             if str(dfexcelData2Abstract.iloc[x, 0]).__contains__(str(dfListFromDb.iloc[y, 0])):
  65.                 # dfexcelData2Abstract2.loc[y,1]=dfListFromDb.loc[y, 1]
  66.                 # print(str(dfexcelData2Abstract2.iloc[x,0])+'#'+str(dfListFromDb.iloc[y,0])+'相等')
  67.                 break
  68.                 # print('dfListFromDb.loc['+str(y)+', 1]:'+str(dfListFromDb.iloc[y, 1]))
  69.                 # print('dfexcelData2Abstract2.loc['+str(x)+', 1]:'+str(dfexcelData2Abstract2.iloc[x, 1]))
  70.                 # print('dfListFromDb.loc['+str(y)+', 2]:'+str(dfListFromDb.iloc[y, 2]))
  71.         # print('dfexcelData2Abstract2.loc['+str(x)+', 2]:'+str(dfexcelData2Abstract2.iloc[x, 1]))
  72.  
  73.     ticks2 = (time.time() - ticks) / 60
  74.     print('ForloopDoubleLIstRunTime 耗时：{}'.format(time.time() - ticks))  # +str(ticks2))
  75.     list24result = []
  76.     # AA='准备补充单号为空的数据:str(list24result[11]).find("基金代发任务"):'+str(list24result[11].find("基金代发任务"))+'#str(list24result[11]).find("订单编号"):'+str(str(list24result[11]).find("订单编号"))+'#str(list24result[21]):'+str(list24result[21])
  77.  
  78. ForloopDoubleLIstRunTime(dfListFromDb, dfexcelData2Abstract2)
  79. print(str(time.process_time()))    #120second 263, 208 132 116 99 98
  80.  
  81. # @ti.func
  82. def DoubleLIstRunTime_iterrows(dfListFromDb, dfexcelData2Abstract2):
  83.     ticks = time.time()
  84.     # dfListFromDb.convert_dtypes()
  85.     # dfexcelData2Abstract2.convert_dtypes()
  86.     for index, dfexcelData2Abstract2_row in dfexcelData2Abstract2.iterrows():
  87.         # 获取用电量和时间（小时）
  88.         # myrow = str(row[0])
  89.         # print('dfexcelData2Abstract2_row:'+str(dfexcelData2Abstract2_row))
  90.         for indexdb,dfListFromDb_rowdb in dfListFromDb.iterrows():
  91.             # print('dfListFromDb_rowdb:'+str(dfListFromDb_rowdb))
  92.             if str(dfexcelData2Abstract2_row[0]).__contains__(str(dfListFromDb_rowdb[0])):
  93.                 print('index:'+str(index)+'str(dfexcelData2Abstract2_row[0]):'+str(dfexcelData2Abstract2_row[0])+'#'+'str(dfListFromDb_rowdb[0]):'+str(dfListFromDb_rowdb[0]))
  94.                 break
  95.         # hour = row['date_time'].hour
  96.         # 添加cost列表
  97.     #     energy_cost = apply_tariff(energy_used, hour)
  98.     #     energy_cost_list.append(energy_cost)
  99.     # df['cost_cents'] = energy_cost_list
  100.     ticks2 = (time.time() - ticks) / 60
  101.     print('DoubleLIstRunTime_iterrows 耗时：{}'.format(time.time() - ticks))  # +str(ticks2))
  102.  
  103. # DoubleLIstRunTime_iterrows(dfListFromDb, dfexcelData2Abstract2)   #123secend
  104. # print(str(time.process_time()))   #377 95  117 168 160
  105.  
  106. # @ti.kernel
  107. def DoubleLIstRunTime_itertuples(dfListFromDb , dfexcelData2Abstract2):
  108.     ticks = time.time()
  109.     # dfListFromDb.convert_dtypes()
  110.     # dfexcelData2Abstract2.convert_dtypes()
  111.     for index, dfexcelData2Abstract2_row in dfexcelData2Abstract2.iterrows():
  112.         # 获取用电量和时间（小时）
  113.         # myrow = str(dfexcelData2Abstract2[0])
  114.         print('外层循环dfexcelData2Abstract2Index:'+str(index)+'#dfexcelData2Abstract2_row:'+str(dfexcelData2Abstract2_row))
  115.         i = 0
  116.         for rowdb in dfListFromDb.itertuples():
  117.             i=i+1
  118.             if str(dfexcelData2Abstract2_row[0]).__contains__(str(getattr(rowdb,'_1'))):
  119.                 # print('内层循环TuplesIndex:'+str(i)+'#str(dfexcelData2Abstract2_row[0]):'+str(dfexcelData2Abstract2_row[0])+'#'+"str(getattr(rowdb['_1'])):"+str(getattr(rowdb,'_1')))
  120.                 break
  121.         # list_itertuples(dfexcelData2Abstract2_row,dfListFromDb)
  122.         # hour = row['date_time'].hour
  123.         # 添加cost列表
  124.     #     energy_cost = apply_tariff(energy_used, hour)
  125.     #     energy_cost_list.append(energy_cost)
  126.     # df['cost_cents'] = energy_cost_list
  127.     ticks2 = (time.time() - ticks) / 60
  128.     print('DoubleLIstRunTime_iterrows 耗时：{}'.format(time.time() - ticks))  # +str(ticks2))
  129.  
  130.  
  131. # DoubleLIstRunTime_itertuples(dfListFromDb, dfexcelData2Abstract2)   #123secend
  132. # print(str(time.process_time()))   #377 95  117 168 160
  133.  
  134. def testfindInApply(tempRow):
  135.     i =0
  136.     # print('外层循环tempRow:' + str(i) + '#str(tempRow):' + str(tempRow) + '#' )
  137.     for rowdb in dfListFromDb.itertuples():
  138.         i = i + 1
  139.         if str(tempRow[0]).__contains__(str(getattr(rowdb, '_1'))):
  140.             # print('内层循环TuplesIndex:'+str(i)+'#str(tempRow[0]):'+str(tempRow[0])+'#'+"str(getattr(rowdb['_1'])):"+str(getattr(rowdb,'_1')))
  141.             tempRow[2]=str(getattr(rowdb, '_1'))
  142.             return True
  143.         # else:
  144.         #     return False
  145.  
  146. # print(dfexcelData2Abstract2)
  147. # dfexcelData2Abstract2[2]='-----------------------------------'
  148. # print(dfexcelData2Abstract2)
  149. # ticks = time.time()
  150. # dfexcelData2Abstract2.apply(testfindInApply,axis=1)
  151. # ticks2 = (time.time() - ticks) / 60
  152. # print('dfexcelData2Abstract2.apply 耗时：{}'.format(time.time() - ticks))  # +str(ticks2))
  153. # print(dfexcelData2Abstract2)
  154. # dfexcelData2Abstract2.to_csv("d:\dfexcelData2Abstract2Output.csv")
  155.

java代码：

  1. package com.company;
  2.  
  3. import javax.swing.*;
  4. import java.io.*;
  5. import java.nio.charset.StandardCharsets;
  6. import java.text.SimpleDateFormat;
  7. import java.util.ArrayList;
  8. import java.util.Arrays;
  9. import java.util.Iterator;
  10. import java.util.List;
  11. import java.io.BufferedReader;
  12. import java.io.FileReader;
  13. import java.io.IOException;
  14. import java.io.BufferedReader;
  15. import java.io.FileReader;
  16. import java.io.IOException;
  17. import java.util.ArrayList;
  18. import java.util.HashMap;
  19. import java.util.List;
  20. import java.util.Map;
  21.  
  22. import static sun.misc.Version.print;
  23. import static sun.misc.Version.println;
  24.  
  25. public class DbListModify {
  26.     static String[][] arrexcelData2Abstract;
  27.     static String[][] arrlistFromDb;
  28.     static File file;
  29.     static File fileCsv;
  30.     static SimpleDateFormat df;
  31.  
  32.     public static void main(String[] args) {
  33.         // write your code here
  34.         int IntCsvexcelData2AbstractLength=0;
  35.         int IntCsvlistFromDbLength=0;
  36.         df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS");
  37.  
  38.  
  39.         try {
  40.             String content = "";
  41.             file =new File("D:\\test_appendfile31"+df.format(System.currentTimeMillis()).toString().replace(":","").replace("/","")+".txt");
  42.             fileCsv=new File("D:\\DbListModify.csv");
  43.             if(!file.exists()){
  44.                 file.createNewFile();
  45.             }
  46.             if(!fileCsv.exists()){
  47.                 file.createNewFile();
  48.             }
  49.             //trueappend file
  50.             FileWriter fileWritter = new FileWriter(file.getAbsoluteFile(),false);//.getAbsoluteFile() getName
  51.             FileWriter CsvfileWritter = new FileWriter(fileCsv.getAbsoluteFile(), false);//.getAbsoluteFile() getName 数据专用文件
  52.             fileWritter.write(content);
  53.  
  54.             //(),
  55.             BufferedReader readerexcelData2Abstract = new BufferedReader(new InputStreamReader(new FileInputStream("d:\\excelData2Abstract.csv"),"gb2312"));//GBK
  56. //                 reader.readLine();//,
  57. //                 System.out.println(reader.readLine());
  58.             String lineexcelData2Abstract = null;
  59.             int i=0;
  60.             while((lineexcelData2Abstract=readerexcelData2Abstract.readLine())!=null){
  61.                 String itemexcelData2Abstract[] = lineexcelData2Abstract.split(",");//CSV抽象文件
  62.                 String last = itemexcelData2Abstract[itemexcelData2Abstract.length-1];//CSV数据库文件
  63.                 i=i+1;
  64.             }
  65.             IntCsvexcelData2AbstractLength=i;
  66.             fileWritter.write("IntCsvexcelData2AbstractLength:"+IntCsvexcelData2AbstractLength+"\r\n");
  67.             readexcelData2AbstractCSV("d:\\excelData2Abstract.csv",IntCsvexcelData2AbstractLength,3);
  68.  
  69.             System.out.println("itemexcelData2Abstract"+df.format(System.currentTimeMillis()).toString());
  70.             fileWritter.write("itemexcelData2Abstract"+df.format(System.currentTimeMillis()).toString()+"\r\n");
  71.             i=0;
  72.             //(),
  73.             BufferedReader readerlistFromDb = new BufferedReader(new InputStreamReader(new FileInputStream("d:\\listFromDb.csv"), "utf-8"));//GBK
  74. //                 reader.readLine();//,
  75. //                 System.out.println(reader.readLine());
  76.             String linelistFromDb = null;
  77.             while((linelistFromDb=readerlistFromDb.readLine())!=null){
  78.                 String itemlistFromDb[] = linelistFromDb.split(",");//CSV,,
  79.                 String last = itemlistFromDb[itemlistFromDb.length-1];//CSV,-1
  80.                 i=i+1;
  81.             }
  82.             IntCsvlistFromDbLength=i;
  83.             fileWritter.write("IntCsvlistFromDbLength:"+IntCsvlistFromDbLength+"\r\n");
  84.  
  85.             readlistFromDbCSV("d:\\listFromDb.csv",IntCsvexcelData2AbstractLength,3);
  86.             System.out.println("arrlistFromDb"+df.format(System.currentTimeMillis()).toString());
  87.             fileWritter.write("CsvexcelData2Abstract赋值开始："+df.format(System.currentTimeMillis()).toString()+"\r\n");
  88.             for(int inta=0;inta<IntCsvexcelData2AbstractLength-2;inta++)
  89.             {
  90.                 for(int intb=0;intb<IntCsvlistFromDbLength;intb++)
  91.                 {
  92.                     if (arrexcelData2Abstract[inta][0].toString().trim().contains(arrlistFromDb[intb][0].toString().trim()))
  93.                     {
  94. //                      System.out.println("arrexcelData2Abstract[inta][0].toString().trim():"+arrexcelData2Abstract[inta][0].toString().trim()+"#arrlistFromDb[intb][0].toString().trim():"+arrlistFromDb[intb][0].toString().trim());
  95.                         arrexcelData2Abstract[inta][2]=arrlistFromDb[intb][1].toString().trim();
  96.                         break;
  97.                     }
  98.                 }
  99.             }
  100.             fileWritter.write("CsvexcelData2Abstract赋值结束：写入CSV开始"+df.format(System.currentTimeMillis()).toString()+"\r\n");
  101.             StringBuilder stringBuilder =new StringBuilder();
  102.             for(int inta=0;inta<IntCsvexcelData2AbstractLength-2;inta++)
  103.             {
  104. //                CsvfileWritter.write(arrexcelData2Abstract[inta][0].toString()+","+arrexcelData2Abstract[inta][1].toString()+","+arrexcelData2Abstract[inta][2].toString()+","+"\r\n");
  105. //                fileWritter.write(""+arrexcelData2Abstract[inta][0].toString()+","+arrexcelData2Abstract[inta][1].toString()+","+arrexcelData2Abstract[inta][2].toString()+","+"\r\n");
  106.                 stringBuilder.append(arrexcelData2Abstract[inta][0].toString()+","+arrexcelData2Abstract[inta][1].toString()+","+arrexcelData2Abstract[inta][2].toString()+","+"\r\n");
  107.                 System.out.println(inta);
  108.             }
  109.             CsvfileWritter.write(stringBuilder.toString());
  110.             fileWritter.write("写入CSV完成"+df.format(System.currentTimeMillis()).toString()+"\r\n");
  111.             fileWritter.flush(); fileWritter.close();
  112.             CsvfileWritter.flush();CsvfileWritter.close();
  113.         } catch (Exception e) {
  114.             e.printStackTrace();
  115.         }
  116.  
  117.     }
  118.  
  119.     private static void readexcelData2AbstractCSV(String pFilename,int IntCsvexcelData2AbstractLength,int intcollen) {
  120.  
  121.         int i=0;
  122.         try {
  123.             BufferedReader br = new BufferedReader(new FileReader(pFilename));
  124.             arrexcelData2Abstract = new String[IntCsvexcelData2AbstractLength][intcollen];
  125. //                BufferedReader br = new BufferedReader(new FileReader(pFilename));
  126.             String line = " ";
  127.             String [] temp;
  128.             while ((line = br.readLine())!= null){
  129.                 temp = line.split(","); //split spaces
  130.                 for (int j = 0; j<3; j++) {
  131.                     if (j==2)
  132.                     {
  133.                         arrexcelData2Abstract[i][j] ="";
  134.                     }
  135.                     else
  136.                         arrexcelData2Abstract[i][j] =temp[j];
  137. //                        System.out.println("readexcelData2AbstractCSVtemp["+j+"]:"+temp[j].toString());
  138.                 }
  139. //                    System.out.println("arr["+i+"]:"+arr[i].toString());
  140.                 System.out.println("i:"+i);
  141. //                    System.out.println("temp:"+temp.);
  142.                 i=i+1;
  143.                 if (i==IntCsvexcelData2AbstractLength-2)
  144.                     return;
  145.             }
  146.  
  147. //            for (int m=0;m<introwlen;m++) {
  148. //                System.out.print(arr[m].toString());
  149. //            }
  150.         }catch (Exception e) {
  151.             e.printStackTrace();
  152.         }
  153.         finally{
  154.         }
  155. //        return arr;
  156.     }
  157.  
  158.     private static void readlistFromDbCSV(String pFilename,int IntCsvlistFromDbLength,int intcollen) {
  159.  
  160.         int i=0;
  161.         try {
  162.             BufferedReader br = new BufferedReader(new FileReader(pFilename));
  163.             arrlistFromDb = new String[IntCsvlistFromDbLength][intcollen];
  164. //                BufferedReader br = new BufferedReader(new FileReader(pFilename));
  165.             String line = " ";
  166.             String [] temp;
  167.             while ((line = br.readLine())!= null){
  168.                 temp = line.split(","); //split spaces
  169.                 for (int j = 0; j<2; j++) {
  170.                     arrlistFromDb[i][j] =temp[j];
  171.                     System.out.println("readlistFromDbCSVtemp["+j+"]:"+temp[j].toString());
  172.                 }
  173. //                    System.out.println("arr["+i+"]:"+arr[i].toString());
  174.                 System.out.println("i:"+i);
  175. //                    System.out.println("temp:"+temp.);
  176.                 i=i+1;
  177. //                if (i==introwlen-2)
  178. //                    return;
  179.             }
  180.  
  181. //            for (int m=0;m<introwlen;m++) {
  182. //                System.out.print(arr[m].toString());
  183. //            }
  184.         }catch (Exception e) {
  185.             e.printStackTrace();
  186.         }
  187.         finally{
  188.         }
  189. //        return arr;
  190.     }
  191.  
  192. }

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/笔触狂放9/article/detail/282037
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号