java大数据文件写下读取分割排序合并.docx

1、java大数据文件写下读取分割排序合并java：大数据文件写下，读取，分割，排序，合并 java：大数据文件写入，读取，分割，排序，合并/* * 大数据排序合并 * * param args */public static void main(String args) throws IOException / 写入文件的路径String filePath = D:456;/ 切分文件的路径String sqlitFilePath = D:456123;/数据的个数int CountNumbers=10000000;/子文件的个数int CountFile=10;/精度int countAccu

2、racy=30*CountFile;long startNumber=System.currentTimeMillis();/ 写入大数据文件WriteData(filePath,CountNumbers);System.out.println(存储完毕);/ 将大数据文件切分到另外的十个小文件中sqlitFileDate(filePath, sqlitFilePath,CountFile);System.out.println(文件切割完毕！);/ 把每个文件的数据进行排序singleFileDataSort(sqlitFilePath,CountFile);System.out.print

3、ln(每个子文件排序完毕！);/精度调整，十个文件数据进行比较整合deathDataFile(filePath,sqlitFilePath,countAccuracy,CountFile);System.out.println(整合完毕);long stopNumber=System.currentTimeMillis();System.out.println(耗时+(stopNumber-startNumber)/1000+毫秒);/ 写入大数据文件public static void WriteData(String path,int CountNumbers) throws IOExce

4、ption path = path + 12114.txt;FileWriter fs = new FileWriter(path);BufferedWriter fw=new BufferedWriter(fs);for (int i = 0; i CountNumbers; i+) fw.write(new Random().nextInt(Integer.MAX_VALUE) + rn);fw.close();fs.close();/ 将大数据文件切分到另外的十个小文件中public static void sqlitFileDate(String filepath, String sq

5、litPath,int CountFile) throws IOException FileWriter fs = null;BufferedWriter fw=null;FileReader fr = new FileReader(filepath + 12114.txt);BufferedReader br = new BufferedReader(fr); / 读取获取整行数据int i = 1;LinkedList WriterLists=new LinkedList(); /初始化文件流对象集合LinkedList fwLists=new LinkedList();for (int

6、j = 1; j = CountFile) i = 1; elsei+;br.close();fr.close();for (Iterator iterator = fwLists.iterator(); iterator.hasNext();) BufferedWriter object = (BufferedWriter) iterator.next();object.close();/遍历关闭所有子文件流for (Iterator iterator = WriterLists.iterator(); iterator.hasNext();) FileWriter object = (Fi

7、leWriter) iterator.next();object.close();/ 把每个文件的数据进行排序public static void singleFileDataSort(String path1,int CountFile) throws IOException LinkedList nums = null;for (int i = 1; i = CountFile; i+) nums = new LinkedList();String path = path1 + 12 + i + .txt;try FileReader fr = new FileReader(path);B

8、ufferedReader br = new BufferedReader(fr);while (br.ready() / 将读取的单个数据加入到集合里面nums.add(Integer.parseInt(br.readLine();/ 对集合进行排序Collections.sort(nums);/ 将排序好的数据写入源文件numberSort(nums, path);br.close();fr.close(); catch (NumberFormatException e) e.printStackTrace(); catch (FileNotFoundException e) e.prin

9、tStackTrace(); catch (IOException e) e.printStackTrace();/ 对每个文件数据进行排序，再写入源文件public static void numberSort(LinkedList list, String path) try FileWriter fs = new FileWriter(path);BufferedWriter fw=new BufferedWriter(fs);for (Iterator iterator = list.iterator(); iterator.hasNext();) Object object = (O

10、bject) iterator.next();fw.write(object + rn);fw.close();fs.close(); catch (IOException e) e.printStackTrace();/ 文件数据最终整合(精度调整)public static void deathDataFile(String filepath, String sqlitFilePath1,int countAccuracy, int CountFile) throws IOException LinkedList nums = new LinkedList(); /添加数据，进行排序Obj

11、ect temp = null; / 记录每次排序剩下的最后一个数字boolean ispass = false;LinkedList ispasses = null; /记录数据文件的状态信息FileWriter fs = new FileWriter(filepath + Sort.txt, false); /创建文件流，以便整合的数据写入BufferedWriter bw=new BufferedWriter(fs);FileReader fr = null; /声明读取文件流BufferedReader br = null; /声明BufferedReaderLinkedList Wr

12、iterLists = new LinkedList(); / 初始化文件流对象集合LinkedList WriterListFile = new LinkedList();for (int j = 1; j 1) / 对集合进行排序Collections.sort(nums);/ 接收最大的数据，其它的数据写入总排序文件temp = numberSortData(nums, filepath, true, countAccuracy,bw);nums = new LinkedList();nums.add(temp);continue;/ 记录下一组数据的位置/ 判断是不是十个文件都没有数据

13、if (ispasses.size() = CountFile) Collections.sort(nums);temp = numberSortData(nums, filepath, true, countAccuracy, bw);nums = new LinkedList();break;bw.close();/关闭写入流fs.close();/关闭所有的BufferedReaderfor (Iterator iterator = WriterLists.iterator(); iterator.hasNext();) BufferedReader object2 = (Buffere

14、dReader) iterator.next();object2.close();/关闭所有的FileReaderfor (Iterator iterator = WriterListFile.iterator(); iterator.hasNext();) FileReader object = (FileReader) iterator.next();object.close();/ 对数据进行排序,写入最终文件中(精度调整)public static Object numberSortData(LinkedList list, String filePath,boolean ispass

15、, int countAccuracy,BufferedWriter fs) Object temp = 0; /记录最后一个值int tempCount = 0; /记录写入的数据位置try for (Iterator iterator = list.iterator(); iterator.hasNext();) Object object = (Object) iterator.next();/ 判断是否是最后一个数if (tempCount = list.size() - 1) / 判断集合里面不足一百個數了if (list.size() countAccuracy + 1 & ispass) temp = null; else temp = object;break;/ 写入数据源fs.write(object + rn);/ 记录数据的下标tempCount+; catch (IOException e) e.printStackTrace();return temp;