三力测试抓取代码

nbjk-admin/src/main/java/com/miaxis/app/controller/question/ThreeForceController.java

@@ -27,10 +27,10 @@ public class ThreeForceController extends BaseController{
     private IThreeForceService threeForceService;
 
     /**
-     * 查询三力测试列表
+     * 查询三力测试列表（一点通）
      */
     @GetMapping("/list")
-    @ApiOperation("查询三力测试列表")
+    @ApiOperation("查询三力测试列表（一点通）")
         @ApiImplicitParams({
             @ApiImplicitParam(name = "pageNum",value = "当前页码" ,dataType = "int", paramType = "query", required = false),
             @ApiImplicitParam(name = "pageSize",value = "每页数据量" , dataType = "int", paramType = "query", required = false),

nbjk-admin/src/test/java/com/miaxis/test/KTDownload.java

@@ -0,0 +1,147 @@
+package com.miaxis.test;
+
+
+import com.miaxis.NbjkApplication;
+import com.miaxis.three.domain.ThreeForce;
+import com.miaxis.three.service.IThreeForceService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.List;
+
+
+@ActiveProfiles("dev")
+@SpringBootTest(classes = NbjkApplication.class)
+@RunWith(SpringRunner.class)
+public class KTDownload {
+
+
+
+
+    @Autowired
+    private IThreeForceService threeForceService;
+
+
+
+
+    @Test
+    public void testExcel() throws Exception {
+
+        //explain_gif
+        List<ThreeForce> threeForceList = threeForceService.selectThreeForceMediaList(); //要修改
+        downLoadList(threeForceList, "ThreeForce");
+
+
+
+//        List<WebNoteAll> webNoteAllList = webNoteAllService.selectWebNoteAllMediaUrlList();
+//        downLoadList2(webNoteAllList, "webNoteAllList");
+
+    }
+
+
+    public static void downLoadList(List<ThreeForce> list, String listType) {
+
+        if ("ThreeForce".equals(listType)) {
+
+            for (int i = 0; i < list.size(); i++) {
+                ThreeForce threeForce = list.get(i);
+                String mediaUrl = threeForce.getMediaUrl(); //要修改
+
+                int index = mediaUrl.lastIndexOf("/");
+                String fileName = mediaUrl.substring(index + 1);
+                System.out.println(mediaUrl + "," + fileName);
+                try {
+                    downLoadFromUrl(mediaUrl, fileName, "G:\\中正\\题库图\\一点通三力图"); //要修改
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+
+
+
+    public static void downLoadFromUrl(String urlStr, String fileName, String savePath) throws IOException, InterruptedException {
+        URL url = new URL(urlStr);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        //设置超时间为3秒
+        conn.setConnectTimeout(3 * 1000);
+        //防止屏蔽程序抓取而返回403错误
+        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
+        //得到输入流
+        InputStream inputStream = conn.getInputStream();
+
+        //获取自己数组
+        byte[] getData = readInputStream(inputStream);
+
+        //文件保存位置
+        File saveDir = new File(savePath);
+        if (!saveDir.exists()) {
+            saveDir.mkdir();
+        }
+        int index = fileName.lastIndexOf(".");
+        String first = fileName.substring(0, index);
+        String lastName = fileName.substring(index);
+        System.out.println(fileName);
+        System.out.println(first);
+        first += lastName; //要修改
+        System.out.println(first);
+
+        File file = new File(saveDir + File.separator + first);
+
+        FileOutputStream fos = new FileOutputStream(file);
+
+        fos.write(getData);
+
+        if (fos != null) {
+            fos.close();
+        }
+        if (inputStream != null) {
+
+            inputStream.close();
+        }
+        Thread.sleep(100);
+        System.out.println("info:" + url + " download success");
+    }
+
+
+    /**
+     * 从输入流中获取字节数组
+     *
+     * @param inputStream
+     * @return
+     * @throws IOException
+     */
+    public static byte[] readInputStream(InputStream inputStream) throws IOException {
+
+        byte[] buffer = new byte[1024];
+
+        int len = 0;
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        while ((len = inputStream.read(buffer)) != -1) {
+            bos.write(buffer, 0, len);
+        }
+        bos.close();
+
+        return bos.toByteArray();
+
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        KTDownload t = new KTDownload();
+        t.testExcel();
+
+    }
+
+
+}

nbjk-admin/src/test/java/com/miaxis/test/Test4.java

@@ -0,0 +1,20 @@
+package com.miaxis.test;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+
+public class Test4 {
+
+    public static void main(String[] args) {
+
+        String encodedString = "%E4%B8%89%E5%8A%9B%E6%B5%8B%E8%AF%95";
+        try {
+            String decodedString = URLDecoder.decode(encodedString, StandardCharsets.UTF_8.toString());
+            System.out.println("Decoded String: " + decodedString);
+        } catch (UnsupportedEncodingException e) {
+            System.err.println("Unsupported Encoding: " + e.getMessage());
+        }
+    }
+
+}

nbjk-admin/src/test/java/com/miaxis/test/WxGzhTest.java

@@ -52,6 +52,11 @@ public class WxGzhTest {
 
     }
 
+    /**
+     * 公众号取读视频链接和图片链接
+     * @throws InterruptedException
+     */
+
     @Test
     public void getWxFreepublish() throws InterruptedException {
         String token = wxGzhService.getGzhToken();

nbjk-service/src/main/java/com/miaxis/three/mapper/ThreeForceMapper.java

@@ -18,7 +18,9 @@ public interface ThreeForceMapper extends BaseMapper<ThreeForce> {
      * @param threeForce 三力测试
      * @return 三力测试集合
      */
-    public List<ThreeForce> selectThreeForceList(ThreeForce threeForce);
+    List<ThreeForce> selectThreeForceList(ThreeForce threeForce);
+
+    List<ThreeForce> selectThreeForceMediaList();
 
     List<ThreeForce> selectTestQuestionInfoList(String sql);
 }

nbjk-service/src/main/java/com/miaxis/three/service/IThreeForceService.java

@@ -22,4 +22,6 @@ public interface IThreeForceService extends IService<ThreeForce>{
     public List<ThreeForce> selectThreeForceList(ThreeForce threeForce);
 
     List<ThreeForce> selectTestQuestionInfoList();
+
+    List<ThreeForce> selectThreeForceMediaList();
 }

nbjk-service/src/main/java/com/miaxis/three/service/impl/ThreeForceServiceImpl.java

@@ -73,4 +73,9 @@ public class ThreeForceServiceImpl extends ServiceImpl<ThreeForceMapper, ThreeFo
             return null;
         }
     }
+
+    @Override
+    public List<ThreeForce> selectThreeForceMediaList() {
+        return threeForceMapper.selectThreeForceMediaList();
+    }
 }

nbjk-service/src/main/resources/mapper/three/ThreeForceMapper.xml

@@ -81,6 +81,11 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
         </where>
     </select>
 
+    <select id="selectThreeForceMediaList"  resultMap="ThreeForceResult">
+        select * from three_force where media_url!=''
+    </select>
+
+
     <select id="selectTestQuestionInfoList" resultType="com.miaxis.three.domain.ThreeForce">
         ${sql}
     </select>