一点通抓取修改

zzjs-admin/src/test/java/com/miaxis/test/KTDownload2.java

@@ -0,0 +1,289 @@
+package com.miaxis.test;
+
+
+import com.miaxis.ZzjsApplication;
+import com.miaxis.question.domain.QuestionInfoKtGet;
+import com.miaxis.question.domain.QuestionInfoKtNew;
+import com.miaxis.question.domain.WebNoteAll;
+import com.miaxis.question.service.IQuestionInfoKtGetService;
+import com.miaxis.question.service.IQuestionInfoKtNewService;
+import com.miaxis.question.service.IWebNoteAllService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.List;
+
+
+@SpringBootTest(classes = ZzjsApplication.class)
+@RunWith(SpringRunner.class)
+public class KTDownload2 {
+
+
+    @Autowired
+    private IQuestionInfoKtGetService questionInfoGetService;
+
+
+    @Autowired
+    private IQuestionInfoKtNewService questionInfoKtNewService;
+
+    @Autowired
+    private IWebNoteAllService webNoteAllService;
+
+
+
+
+    @Test
+    public void testExcel() throws Exception {
+
+        //explain_gif
+        List<QuestionInfoKtNew> explainGifList = questionInfoKtNewService.selectQuestionInfoKtNewExplainGifList(); //要修改
+        downLoadList(explainGifList, "explainGifList");
+
+        //explain_mp3
+        List<QuestionInfoKtNew> explainMp3List = questionInfoKtNewService.selectQuestionInfoKtNewExplainMp3List(); //要修改
+        downLoadList(explainMp3List, "explainMp3List");
+
+        //image
+        List<QuestionInfoKtNew> imageList = questionInfoKtNewService.selectQuestionInfoKtNewImageList(); //要修改
+        downLoadList(imageList, "imageList");
+
+        //image_ydt
+        List<QuestionInfoKtNew> imageYdtList = questionInfoKtNewService.selectQuestionInfoKtNewImageYdtList(); //要修改
+        downLoadList(imageYdtList, "imageYdtList");
+
+        //issuemp3
+        List<QuestionInfoKtNew> issuemp3List = questionInfoKtNewService.selectQuestionInfoKtNewIssuemp3List(); //要修改
+        downLoadList(issuemp3List, "issuemp3List");
+
+
+        //answermp3
+        List<QuestionInfoKtNew> answermp3List = questionInfoKtNewService.selectQuestionInfoKtNewAnswermp3List(); //要修改
+        downLoadList(answermp3List, "answermp3List");
+
+        //explainjsmp3
+        List<QuestionInfoKtNew> explainjsmp3List = questionInfoKtNewService.selectQuestionInfoKtNewExplainjsmp3List(); //要修改
+        downLoadList(explainjsmp3List, "explainjsmp3List");
+
+
+//        List<WebNoteAll> webNoteAllList = webNoteAllService.selectWebNoteAllMediaUrlList();
+//        downLoadList2(webNoteAllList, "webNoteAllList");
+
+    }
+
+
+    public static void downLoadList2(List<WebNoteAll> list, String listType) {
+
+        if ("webNoteAllList".equals(listType)) {
+
+            for (int i = 0; i < list.size(); i++) {
+                WebNoteAll WebNoteAll = list.get(i);
+                String mediaUrl = WebNoteAll.getMediaUrl(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20230223\\media_ydt"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+
+    public static void downLoadList(List<QuestionInfoKtNew> list, String listType) {
+
+        if ("explainGifList".equals(listType)) {
+
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getExplainGif(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\explain_gif"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+
+        } else if ("explainMp3List".equals(listType)) {
+
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getExplainMp3(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\explain_mp3"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+        } else if ("imageList".equals(listType)) {
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getImage(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\image"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        } else if("imageYdtList".equals(listType)){
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getImageYdt(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\image_ydt"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }  else if ("issuemp3List".equals(listType)) {
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getIssuemp3(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\issue_mp3"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        } else if ("answermp3List".equals(listType)) {
+
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getAnswermp3(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\answer_mp3"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+        } else if ("explainjsmp3List".equals(listType)) {
+            for (int i = 0; i < list.size(); i++) {
+                QuestionInfoKtNew questionInfoKtNew = list.get(i);
+                String mediaUrl = questionInfoKtNew.getExplainjsmp3(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\kt20231106\\explain_js_mp3");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+
+
+    }
+
+
+    public static void downLoadFromUrl(String urlStr, String fileName, String savePath) throws IOException, InterruptedException {
+        URL url = new URL(urlStr);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        //设置超时间为3秒
+        conn.setConnectTimeout(3 * 1000);
+        //防止屏蔽程序抓取而返回403错误
+        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
+        //得到输入流
+        InputStream inputStream = conn.getInputStream();
+
+        //获取自己数组
+        byte[] getData = readInputStream(inputStream);
+
+        //文件保存位置
+        File saveDir = new File(savePath);
+        if (!saveDir.exists()) {
+            saveDir.mkdir();
+        }
+        int index = fileName.lastIndexOf(".");
+        String first = fileName.substring(0, index);
+        String lastName = fileName.substring(index);
+        System.out.println(fileName);
+        System.out.println(first);
+        first += lastName; //要修改
+        System.out.println(first);
+
+        File file = new File(saveDir + File.separator + first);
+
+        FileOutputStream fos = new FileOutputStream(file);
+
+        fos.write(getData);
+
+        if (fos != null) {
+            fos.close();
+        }
+        if (inputStream != null) {
+
+            inputStream.close();
+        }
+        Thread.sleep(100);
+        System.out.println("info:" + url + " download success");
+    }
+
+
+    /**
+     * 从输入流中获取字节数组
+     *
+     * @param inputStream
+     * @return
+     * @throws IOException
+     */
+    public static byte[] readInputStream(InputStream inputStream) throws IOException {
+
+        byte[] buffer = new byte[1024];
+
+        int len = 0;
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        while ((len = inputStream.read(buffer)) != -1) {
+            bos.write(buffer, 0, len);
+        }
+        bos.close();
+
+        return bos.toByteArray();
+
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        KTDownload2 t = new KTDownload2();
+        t.testExcel();
+
+    }
+
+
+}