moving from old svn to git
This commit is contained in:
commit
7da1050337
|
|
@ -0,0 +1,47 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="owner.project.facets" value="jst.java"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jst.server.core.container/org.eclipse.jst.server.tomcat.runtimeTarget/Apache Tomcat v6.0">
|
||||
<attributes>
|
||||
<attribute name="owner.project.facets" value="jst.web"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.web.container"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.module.container"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/org.json.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/gson-2.2.3.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/bonecp-0.7.1.RELEASE.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/slf4j-api-1.7.5.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/slf4j-ext-1.7.5.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/guava-14.0.1.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-dbutils-1.5.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/log4j-1.2.17.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/slf4j-nop-1.7.5.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/mysql-connector-java-5.1.22-bin.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/apache-mime4j-benchmark-0.7.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/apache-mime4j-core-0.7.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/apache-mime4j-dom-0.7.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/apache-mime4j-examples-0.7.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/apache-mime4j-storage-0.7.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-io-1.4.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-logging-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/msgparser-1.11.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-digester3-3.2-with-deps.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/sqljet-1.1.7.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/svnkit-1.7.9.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/svnkit-cli-1.7.9.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/svnkit-javahl16-1.7.9.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/trilead-ssh2-1.0.0-build216.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/antlr-runtime-3.4.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-httpclient-3.0.1.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/commons-codec-1.2.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/jcifs-1.3.17.jar"/>
|
||||
<classpathentry kind="lib" path="WebContent/WEB-INF/lib/filterbuilder.jar"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
|
||||
<classpathentry kind="output" path="build/classes"/>
|
||||
</classpath>
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>EOS</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.wst.common.project.facet.core.builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.wst.validation.validationbuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
|
||||
<nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
|
||||
<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.JRE_CONTAINER"/>
|
||||
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.WebProject">
|
||||
<attributes>
|
||||
<attribute name="hide" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.baseBrowserLibrary"/>
|
||||
<classpathentry kind="output" path=""/>
|
||||
</classpath>
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding/<project>=UTF-8
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
#Sat Jul 06 22:30:16 CST 2013
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.source=1.6
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project-modules id="moduleCoreId" project-version="1.5.0">
|
||||
<wb-module deploy-name="EOS">
|
||||
<wb-resource deploy-path="/" source-path="/WebContent"/>
|
||||
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src"/>
|
||||
<property name="java-output-path"/>
|
||||
<property name="context-root" value="EOS"/>
|
||||
</wb-module>
|
||||
</project-modules>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<faceted-project>
|
||||
<runtime name="Apache Tomcat v6.0"/>
|
||||
<fixed facet="jst.java"/>
|
||||
<fixed facet="jst.web"/>
|
||||
<installed facet="jst.java" version="6.0"/>
|
||||
<installed facet="jst.web" version="2.5"/>
|
||||
<installed facet="wst.jsdt.web" version="1.0"/>
|
||||
</faceted-project>
|
||||
|
|
@ -0,0 +1 @@
|
|||
org.eclipse.wst.jsdt.launching.baseBrowserLibrary
|
||||
|
|
@ -0,0 +1 @@
|
|||
Window
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
Manifest-Version: 1.0
|
||||
Class-Path:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,63 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" id="WebApp_ID" version="2.5">
|
||||
<display-name>EOS</display-name>
|
||||
<welcome-file-list>
|
||||
<welcome-file>index.html</welcome-file>
|
||||
<welcome-file>index.htm</welcome-file>
|
||||
<welcome-file>index.jsp</welcome-file>
|
||||
<welcome-file>default.html</welcome-file>
|
||||
<welcome-file>default.htm</welcome-file>
|
||||
<welcome-file>default.jsp</welcome-file>
|
||||
</welcome-file-list>
|
||||
<servlet>
|
||||
<description></description>
|
||||
<display-name>InitServlet</display-name>
|
||||
<servlet-name>InitServlet</servlet-name>
|
||||
<servlet-class>cn.edu.pku.EOS.servlet.InitServlet</servlet-class>
|
||||
<load-on-startup>1</load-on-startup>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>InitServlet</servlet-name>
|
||||
<url-pattern>/InitServlet</url-pattern>
|
||||
</servlet-mapping>
|
||||
<servlet>
|
||||
<description></description>
|
||||
<display-name>Test</display-name>
|
||||
<servlet-name>Test</servlet-name>
|
||||
<servlet-class>cn.edu.pku.EOS.servlet.Test</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>Test</servlet-name>
|
||||
<url-pattern>/Test</url-pattern>
|
||||
</servlet-mapping>
|
||||
<servlet>
|
||||
<description></description>
|
||||
<display-name>CrawlAllResource</display-name>
|
||||
<servlet-name>CrawlAllResource</servlet-name>
|
||||
<servlet-class>cn.edu.pku.EOS.servlet.CrawlAllResource</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>CrawlAllResource</servlet-name>
|
||||
<url-pattern>/CrawlAllResource</url-pattern>
|
||||
</servlet-mapping>
|
||||
<servlet>
|
||||
<description></description>
|
||||
<display-name>StartCrawlerTask</display-name>
|
||||
<servlet-name>StartCrawlerTask</servlet-name>
|
||||
<servlet-class>cn.edu.pku.EOS.servlet.StartCrawlerTask</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>StartCrawlerTask</servlet-name>
|
||||
<url-pattern>/StartCrawlerTask</url-pattern>
|
||||
</servlet-mapping>
|
||||
<servlet>
|
||||
<description></description>
|
||||
<display-name>GetTaskStatus</display-name>
|
||||
<servlet-name>GetTaskStatus</servlet-name>
|
||||
<servlet-class>cn.edu.pku.EOS.servlet.GetTaskStatus</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>GetTaskStatus</servlet-name>
|
||||
<url-pattern>/GetTaskStatus</url-pattern>
|
||||
</servlet-mapping>
|
||||
</web-app>
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
/cn
|
||||
/log4j.properties
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package cn.edu.pku.EOS.DAO;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import cn.edu.pku.EOS.entity.CrawlerTask;
|
||||
|
||||
public class CrawlerTaskDao {
|
||||
|
||||
|
||||
public static int insertCrawlerTask(CrawlerTask crawlerTask) throws SQLException {
|
||||
int result = DAOUtils.update("INSERT INTO crawlerTask (uuid, projectUuid, crawlerNode, resourceType, startTime) VALUES (?,?,?,?,?)",
|
||||
crawlerTask.getUuid(), crawlerTask.getProjectUuid(), crawlerTask.getCrawlerNode(), crawlerTask.getResourceType(), crawlerTask.getStartTime());
|
||||
return result;
|
||||
}
|
||||
|
||||
public static List<CrawlerTask> getUnfinishedCrawlerTask() throws SQLException{
|
||||
List<CrawlerTask> crawlerTasks = DAOUtils.getResult(CrawlerTask.class, "select * from crawlerTask where status = ?", 0);
|
||||
return crawlerTasks;
|
||||
}
|
||||
|
||||
public static int updateTaskStatus(String uuid, int status, Date finishTime) throws SQLException {
|
||||
int result = DAOUtils.update("UPDATE crawlerTask SET status = ?, finishTime = ? WHERE uuid = ?",
|
||||
status, finishTime, uuid);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
public static CrawlerTask getTaskByUuid(String taskuuid) throws SQLException{
|
||||
List<CrawlerTask> crawlerTasks = DAOUtils.getResult(CrawlerTask.class, "select * from crawlerTask where uuid = ?", taskuuid);
|
||||
return crawlerTasks.get(0);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
JDBCPool.initPool();
|
||||
CrawlerTask c1 = new CrawlerTask();
|
||||
c1.setCrawlerNode("1");
|
||||
c1.setResourceType("hehe");
|
||||
c1.setProjectUuid("haha-gaga");
|
||||
CrawlerTask c2 = new CrawlerTask();
|
||||
c2.setCrawlerNode("2");
|
||||
c2.setResourceType("gege");
|
||||
c2.setProjectUuid("gaga-gaga");
|
||||
|
||||
insertCrawlerTask(c1);
|
||||
insertCrawlerTask(c2);
|
||||
|
||||
List<CrawlerTask> crawlerTasks = getUnfinishedCrawlerTask();
|
||||
for (CrawlerTask crawlerTask : crawlerTasks) {
|
||||
System.out.println(crawlerTask.getStartTime());
|
||||
}
|
||||
|
||||
JDBCPool.shutDown();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package cn.edu.pku.EOS.DAO;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.dbutils.DbUtils;
|
||||
import org.apache.commons.dbutils.QueryRunner;
|
||||
import org.apache.commons.dbutils.ResultSetHandler;
|
||||
import org.apache.commons.dbutils.handlers.BeanListHandler;
|
||||
|
||||
import cn.edu.pku.EOS.TestUtil;
|
||||
|
||||
/**
|
||||
* 存放数据库读取公用方法的公共类
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
|
||||
public class DAOUtils {
|
||||
|
||||
|
||||
public static Connection getConnection() throws SQLException, ClassNotFoundException {
|
||||
return JDBCPool.getConnection();
|
||||
}
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* select语句查询，得到的结果直接包装成javaBean形式，保证你定义的T类型中所有和数据库同名的域都有getter和setter方法
|
||||
* 参见main方法
|
||||
*/
|
||||
public static <T> List<T> getResult(Class<T> t, String sqlString, Object... params) throws SQLException {
|
||||
Connection connection2 = JDBCPool.getConnection();
|
||||
ResultSetHandler<List<T>> h = new BeanListHandler<T>(t);
|
||||
QueryRunner runner = new QueryRunner();
|
||||
List<T> result = runner.query(connection2, sqlString, h, params);
|
||||
DbUtils.close(connection2);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 同步执行update语句
|
||||
* 参见main方法
|
||||
*/
|
||||
public static int update(String sqlString, Object... params) throws SQLException {
|
||||
Connection connection2 = JDBCPool.getConnection();
|
||||
QueryRunner runner = new QueryRunner();
|
||||
int result = runner.update(connection2, sqlString, params);
|
||||
DbUtils.close(connection2);
|
||||
return result;
|
||||
}
|
||||
// /**
|
||||
// * @author 张灵箫
|
||||
// * 异步执行update语句
|
||||
// * 有大量写操作时建议使用异步方法
|
||||
// */
|
||||
// public static void AsyncUpdate(Connection connection, String sqlString, Object... params) {
|
||||
// ExecutorCompletionService<Integer> executor =
|
||||
// new ExecutorCompletionService<Integer>(Executors.newCachedThreadPool());
|
||||
// AsyncQueryRunner asyncRun = new AsyncQueryRunner(executor);
|
||||
//
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(TestUtil.getLuceneProject().getDescription());
|
||||
System.out.println("done!");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package cn.edu.pku.EOS.DAO;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
|
||||
import cn.edu.pku.EOS.config.Config;
|
||||
import com.jolbox.bonecp.BoneCP;
|
||||
import com.jolbox.bonecp.BoneCPConfig;
|
||||
/**
|
||||
* 用单件模式建立数据库连接池的类
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class JDBCPool {
|
||||
private static BoneCP connectionPool = null;
|
||||
|
||||
private JDBCPool() throws SQLException{
|
||||
|
||||
}
|
||||
|
||||
public static Connection getConnection() throws SQLException{
|
||||
// if (connectionPool == null) {
|
||||
// System.out.println("DB pool is disconnected!\n用main class测试数据库时请先调用JDBCPool.init方法。");
|
||||
// return null;
|
||||
// //initPool();
|
||||
// }
|
||||
return connectionPool.getConnection();
|
||||
}
|
||||
|
||||
public static void shutDown() {
|
||||
if (connectionPool != null) {
|
||||
connectionPool.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
public static synchronized void initPool() throws SQLException, ClassNotFoundException{
|
||||
System.out.println("initiating db pool...");
|
||||
if (connectionPool != null) {
|
||||
return;
|
||||
}
|
||||
Class.forName("com.mysql.jdbc.Driver");
|
||||
// setup the connection pool
|
||||
BoneCPConfig config = new BoneCPConfig();
|
||||
config.setJdbcUrl(Config.getDbhost()); // jdbc url specific to your
|
||||
// database, eg
|
||||
// jdbc:mysql://127.0.0.1/yourdb
|
||||
config.setUsername(Config.getDbuser());
|
||||
config.setPassword(Config.getDbpw());
|
||||
config.setMinConnectionsPerPartition(5);
|
||||
config.setMaxConnectionsPerPartition(10);
|
||||
config.setPartitionCount(1);
|
||||
connectionPool = new BoneCP(config); // setup the connection pool
|
||||
System.out.println("db pool initiated!");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
JDBCPool.initPool();
|
||||
Connection connection = JDBCPool.getConnection();
|
||||
System.out.println(connection);
|
||||
} catch (SQLException e) {
|
||||
System.out.println("connection pool initiation fail!");
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
System.out.println("connection pool initiation fail!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package cn.edu.pku.EOS.DAO;
|
||||
|
||||
import java.sql.SQLException;
|
||||
|
||||
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
|
||||
public class ProjectDAO {
|
||||
|
||||
|
||||
public Project getProjectByUuid(String uuid) throws SQLException, NullPointerException{
|
||||
Project project = DAOUtils.getResult(Project.class, "select * from project where uuid = ?", uuid).get(0);
|
||||
return project;
|
||||
}
|
||||
|
||||
public Project getProjectByName(String projectName) throws SQLException, IndexOutOfBoundsException{
|
||||
Project project = DAOUtils.getResult(Project.class, "select * from project where name = ?", projectName).get(0);
|
||||
return project;
|
||||
}
|
||||
|
||||
public int updateProjectInfo(String uuid, Project project) throws SQLException {
|
||||
int result = DAOUtils.update("UPDATE project SET name = ?, hostUrl = ?, programmingLanguage = ?, description = ? WHERE uuid = ?",
|
||||
project.getName(), project.getHostUrl(), project.getProgrammingLanguage(), project.getDescription(), uuid);
|
||||
return result;
|
||||
}
|
||||
|
||||
public int insertProject(Project project) throws SQLException {
|
||||
int result = DAOUtils.update("INSERT INTO project (uuid, name, programmingLanguage, description, hostUrl) VALUES (?,?,?,?,?)",
|
||||
project.getUuid(), project.getName(), project.getProgrammingLanguage(), project.getDescription(), project.getHostUrl());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
JDBCPool.initPool();
|
||||
System.out.println(new ProjectDAO().getProjectByUuid("b7914db3-caa7-4d70-96cd-bd4b5b4ed029").getDescription());
|
||||
JDBCPool.shutDown();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package cn.edu.pku.EOS.DAO;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
|
||||
public class ResourceMetaDataDAO {
|
||||
|
||||
public List<ResourceMetaData> getDataListByProjectUuid(String uuid) throws SQLException {
|
||||
List<ResourceMetaData> datas = DAOUtils.getResult(ResourceMetaData.class, "select * from resourceMetaData where projectuuid = ?", uuid);
|
||||
for (ResourceMetaData resourceMetaData : datas) {
|
||||
//System.out.println(resourceMetaData.getType());
|
||||
resourceMetaData.getUrlsFromUrlString(resourceMetaData.getUrlListString());
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
public int insertResourceMetaData(String projectuuid, ResourceMetaData data) throws SQLException {
|
||||
int result = DAOUtils.update("INSERT INTO resourceMetaData(projectuuid, type, crawler, urlListString) " +
|
||||
"VALUES(?, ?, ?, ?)", projectuuid, data.getType(), data.getCrawler(), data.getUrlListString());
|
||||
return result;
|
||||
}
|
||||
|
||||
public int deleteAllMetaDataOfProject(String uuid) throws SQLException {
|
||||
int result = DAOUtils.update("DELETE FROM resourceMetaData WHERE projectuuid = ?", uuid);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package cn.edu.pku.EOS;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import cn.edu.pku.EOS.DAO.JDBCPool;
|
||||
import cn.edu.pku.EOS.business.ProjectBusiness;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
|
||||
/**
|
||||
* 测试类
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class TestUtil {
|
||||
|
||||
/**
|
||||
* 本地用main方法测试时，可以用该方法得到一个lucene的project对象
|
||||
* @author 张灵箫
|
||||
* @throws ClassNotFoundException
|
||||
* @throws SQLException
|
||||
*/
|
||||
public static Project getLuceneProject(){
|
||||
//插入lucene项目
|
||||
//Project luceneProject = new Project("Apache Lucene");
|
||||
//DAOUtils.update(connection, "INSERT INTO project (uuid, name, programmingLanguage, description, hostUrl) VALUES (?,?,?,?,?)",
|
||||
// luceneProject.getUuid(), luceneProject.getName(),Project.JAVA , "Apache LuceneTM is a high-performance, full-featured text search engine library written entirely in Java. It is a technology suitable for nearly any application that requires full-text search, especially cross-platform.", "http://lucene.apache.org/core/" );
|
||||
try {
|
||||
JDBCPool.initPool();
|
||||
ProjectBusiness pBusiness = new ProjectBusiness();
|
||||
return pBusiness.getProjectByUuid("b7914db3-caa7-4d70-96cd-bd4b5b4ed029");
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
JDBCPool.shutDown();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 打印项目全部信息
|
||||
* @param project
|
||||
*/
|
||||
public static void printProjectInfo(Project project) {
|
||||
System.out.println("==================this is project info=================");
|
||||
System.out.println(project.getName());
|
||||
System.out.println(project.getHostUrl());
|
||||
System.out.println(project.getProgrammingLanguage());
|
||||
System.out.println(project.getDescription());
|
||||
List<ResourceMetaData> datas = project.getResources();
|
||||
System.out.println(datas.size());
|
||||
for (ResourceMetaData resourceMetaData : datas) {
|
||||
System.out.println(resourceMetaData.getType());
|
||||
System.out.println(resourceMetaData.getCrawler());
|
||||
List<String> urlList= resourceMetaData.getBaseUrls();
|
||||
for (String url : urlList) {
|
||||
System.out.println(url);
|
||||
}
|
||||
}
|
||||
System.out.println("==================this is project info=================");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
//String aString = "a\\?b";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package cn.edu.pku.EOS.business;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import cn.edu.pku.EOS.DAO.JDBCPool;
|
||||
import cn.edu.pku.EOS.config.Config;
|
||||
|
||||
/**
|
||||
* 服务器启动时处理初始化事务的类，包括初始化数据库连接池等
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
|
||||
public class InitBusiness {
|
||||
public static void initEOS(){
|
||||
try {
|
||||
JDBCPool.initPool();
|
||||
File file = new File(Config.getTempDir());
|
||||
if (!file.exists()) {
|
||||
file.mkdir();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("initiation failed!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
package cn.edu.pku.EOS.business;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.eclipse.jdt.internal.compiler.ast.ThisReference;
|
||||
|
||||
import cn.edu.pku.EOS.TestUtil;
|
||||
import cn.edu.pku.EOS.DAO.CrawlerTaskDao;
|
||||
import cn.edu.pku.EOS.DAO.ProjectDAO;
|
||||
import cn.edu.pku.EOS.DAO.ResourceMetaDataDAO;
|
||||
import cn.edu.pku.EOS.entity.CrawlerTask;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
import cn.edu.pku.EOS.exception.CrawlerNodeNotRespondingException;
|
||||
import cn.edu.pku.EOS.exception.CrawlerNotInNodeListException;
|
||||
import cn.edu.pku.EOS.nodedispatch.JobDispatcher;
|
||||
import cn.edu.pku.EOS.nodedispatch.MessageSender;
|
||||
import cn.edu.pku.EOS.profileGenerator.ProjectMetaDataParser;
|
||||
import cn.edu.pku.EOS.profileGenerator.ProjectMetaDataParserFactory;
|
||||
/**
|
||||
* project相关业务逻辑
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class ProjectBusiness {
|
||||
|
||||
|
||||
private static final Logger logger = Logger.getLogger(ThisReference.class.getName());
|
||||
/**
|
||||
* 根据uuid得到project对象，包含所有的resourceMetadata
|
||||
* @author 张灵箫
|
||||
* @param uuid
|
||||
* @return
|
||||
*/
|
||||
public Project getProjectByUuid(String uuid) {
|
||||
Project project = null;
|
||||
List<ResourceMetaData> rDatas = null;
|
||||
ProjectDAO projectDAO = new ProjectDAO();
|
||||
ResourceMetaDataDAO resourceMetaDataDAO = new ResourceMetaDataDAO();
|
||||
try {
|
||||
project = projectDAO.getProjectByUuid(uuid);
|
||||
rDatas = resourceMetaDataDAO.getDataListByProjectUuid(uuid);
|
||||
project.getResources().addAll(rDatas);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
return project;
|
||||
}
|
||||
/**
|
||||
* 根据用户上传的XML文件更改已有项目的信息，包括资源信息
|
||||
* XML上传到项目对应的文件夹中，以uuid命名
|
||||
* @param uuid
|
||||
* @return 1表示成功，0表示出错
|
||||
*/
|
||||
public int updateProjectInfo(Project project, String uuid) {
|
||||
ProjectDAO pDao = new ProjectDAO();
|
||||
ResourceMetaDataDAO rDao = new ResourceMetaDataDAO();
|
||||
try {
|
||||
//System.out.println(project.getDescription());
|
||||
pDao.updateProjectInfo(uuid, project);
|
||||
rDao.deleteAllMetaDataOfProject(uuid);
|
||||
for (ResourceMetaData data : project.getResources()) {
|
||||
rDao.insertResourceMetaData(uuid, data);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
System.out.println("Database error!");
|
||||
e.printStackTrace();
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 调用爬虫爬去某个项目的某个指定类型的资源
|
||||
* @param string
|
||||
* @param project
|
||||
*/
|
||||
public String startNewCrawlerTask(String resoucetype, Project project) {
|
||||
ResourceMetaData data = project.getResourceByType(resoucetype);
|
||||
JobDispatcher jd;
|
||||
try {
|
||||
jd = new JobDispatcher(data.getCrawler());
|
||||
} catch (CrawlerNotInNodeListException e) {
|
||||
logger.info(data.getCrawler() + " do not have a node!");
|
||||
e.printStackTrace();
|
||||
return data.getCrawler() + " do not have a node!";
|
||||
}
|
||||
String result;
|
||||
CrawlerTask crawlerTask = new CrawlerTask(project, resoucetype);
|
||||
try {
|
||||
result = jd.dispatchCrawlerJob(crawlerTask);
|
||||
} catch (CrawlerNodeNotRespondingException e) {
|
||||
logger.info(e.getMessage());
|
||||
e.printStackTrace();
|
||||
return e.getMessage();
|
||||
}
|
||||
if (result.equals(MessageSender.REMOTE_SUCCESS_RESPONCE)) {
|
||||
try {
|
||||
CrawlerTaskDao.insertCrawlerTask(crawlerTask);
|
||||
} catch (SQLException e) {
|
||||
logger.info(e.getMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
//System.out.println(data);
|
||||
// Crawler crawler = null;
|
||||
// try {
|
||||
// //System.out.println(Crawler.class.getPackage().getName() + "." + data.getCrawler());
|
||||
// crawler = (Crawler) Class.forName(Crawler.class.getPackage().getName() + "." + data.getCrawler()).newInstance();
|
||||
// crawler.setProject(project);
|
||||
// crawler.setUrlList(data.getBaseUrls());
|
||||
// } catch (Exception e) {
|
||||
// System.out.println("创建爬虫对象时错误!");
|
||||
// e.printStackTrace();
|
||||
// return;
|
||||
// }
|
||||
// CrawlerPool.startCrawlerThread(crawler);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* project对象，存入数据库
|
||||
* @param xmlFilePath xml文件路径
|
||||
* @return
|
||||
*/
|
||||
public void createNewProject(Project project) {
|
||||
ProjectDAO projectDAO = new ProjectDAO();
|
||||
ResourceMetaDataDAO dataDAO = new ResourceMetaDataDAO();
|
||||
try {
|
||||
projectDAO.insertProject(project);
|
||||
List<ResourceMetaData> datas = project.getResources();
|
||||
for (ResourceMetaData resourceMetaData : datas) {
|
||||
dataDAO.insertResourceMetaData(project.getUuid(), resourceMetaData);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.info("Database error!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public Project parseProjectInfoFromWebPage(String pageurl) {
|
||||
Project project = null;
|
||||
String parsertype = ProjectMetaDataParserFactory.inferParserTypeFromUrl(pageurl);
|
||||
ProjectMetaDataParser parser = ProjectMetaDataParserFactory.getParser(parsertype);
|
||||
try {
|
||||
project = parser.parseProjectInfoFromWebPage(pageurl);
|
||||
} catch (IOException e) {
|
||||
logger.info("Unable to parse page to Project: network failure!");
|
||||
e.printStackTrace();
|
||||
} catch (NullPointerException e) {
|
||||
logger.info("Unable to parse page to Project: Unkown type!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
public Project getProjectByName(String name) {
|
||||
ProjectDAO projectDAO = new ProjectDAO();
|
||||
try {
|
||||
return projectDAO.getProjectByName(name);
|
||||
} catch (IndexOutOfBoundsException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
ProjectBusiness pb = new ProjectBusiness();
|
||||
Project project = pb.parseProjectInfoFromWebPage("http://projects.apache.org/projects/lucene_core.html");
|
||||
TestUtil.printProjectInfo(project);
|
||||
//pb.createNewProject(project);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package cn.edu.pku.EOS.business;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.eclipse.jdt.internal.compiler.ast.ThisReference;
|
||||
|
||||
import cn.edu.pku.EOS.DAO.CrawlerTaskDao;
|
||||
import cn.edu.pku.EOS.entity.CrawlerTask;
|
||||
import cn.edu.pku.EOS.exception.CrawlerNodeNotRespondingException;
|
||||
import cn.edu.pku.EOS.nodedispatch.KVPair;
|
||||
import cn.edu.pku.EOS.nodedispatch.MessageSender;
|
||||
import cn.edu.pku.EOS.nodedispatch.NodeManager;
|
||||
/**
|
||||
* 查看爬去任务状态的相关处理逻辑
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class TaskBusiness {
|
||||
private static final Logger logger = Logger.getLogger(ThisReference.class.getName());
|
||||
|
||||
|
||||
/**
|
||||
* 查看任务状态，从数据库读取状态
|
||||
* 如果正在进行，那么要查看子节点是否有该正在运行
|
||||
* 没有或者无法连接搜算任务异常终止
|
||||
* @author 张灵箫
|
||||
* @param taskuuid
|
||||
* @return
|
||||
*/
|
||||
public static int checkTaskStatus(String taskuuid) {
|
||||
CrawlerTask crawlerTask;
|
||||
try {
|
||||
crawlerTask = CrawlerTaskDao.getTaskByUuid(taskuuid);
|
||||
} catch (SQLException e) {
|
||||
logger.info("db error: retrieving CrawlerTask failed!");
|
||||
e.printStackTrace();
|
||||
return -1;
|
||||
}
|
||||
if (crawlerTask.getStatus() != 0) {
|
||||
return crawlerTask.getStatus();
|
||||
}
|
||||
List<KVPair> params = new ArrayList<KVPair>();
|
||||
params.add(new KVPair("taskuuid", taskuuid));
|
||||
try {
|
||||
String result = MessageSender.sendCommand(crawlerTask.getCrawlerNode(), NodeManager.getCheckTaskCommand(), params);
|
||||
if (result.equals("true")) {
|
||||
return CrawlerTask.WAITING;
|
||||
} else {
|
||||
CrawlerTaskDao.updateTaskStatus(taskuuid, CrawlerTask.ERROR, null);
|
||||
}
|
||||
} catch (CrawlerNodeNotRespondingException e) {
|
||||
e.printStackTrace();
|
||||
try {
|
||||
CrawlerTaskDao.updateTaskStatus(taskuuid, CrawlerTask.ERROR, null);
|
||||
} catch (SQLException e1) {
|
||||
logger.info("db error: cannot report error task!");
|
||||
e1.printStackTrace();
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.info("db error: cannot report error task!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
return CrawlerTask.ERROR;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package cn.edu.pku.EOS.config;
|
||||
|
||||
import java.util.ResourceBundle;
|
||||
/**
|
||||
*
|
||||
* @author 灵箫
|
||||
* 读取配置文件的类
|
||||
* 读值请调用getValue(String key)
|
||||
*/
|
||||
public class Config {
|
||||
// 属性文件标识符
|
||||
private static String CONFIG_FILE_NAME = "cn.edu.pku.EOS.config.eos";
|
||||
|
||||
// 所使用的ResourceBundle
|
||||
private static ResourceBundle bundle;
|
||||
|
||||
// 静态私有方法，用于从属性文件中取得属性值
|
||||
static {
|
||||
try {
|
||||
//System.out.println("init config file!");
|
||||
bundle = ResourceBundle.getBundle(CONFIG_FILE_NAME);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String getValue(String key) {
|
||||
return bundle.getString(key);
|
||||
}
|
||||
|
||||
public static String getDbhost() {
|
||||
return getValue("dbhost");
|
||||
}
|
||||
|
||||
public static String getDbuser() {
|
||||
return getValue("dbuser");
|
||||
}
|
||||
|
||||
public static String getDbpw() {
|
||||
return getValue("dbpw");
|
||||
}
|
||||
|
||||
public static String getTempDir() {
|
||||
return getValue("tmpdir");
|
||||
}
|
||||
|
||||
public static String getEOSDir() {
|
||||
return getValue("eosdir");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(getDbhost());
|
||||
System.out.println(getDbuser());
|
||||
System.out.println(getDbpw());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#database
|
||||
dbhost=jdbc:mysql://192.168.4.182:3306/EOS
|
||||
dbuser=root
|
||||
dbpw=woxnsk
|
||||
|
||||
#filedir
|
||||
eosdir=smb://192.168.4.9/Upload/EOS/
|
||||
tmpdir=c:/tmp
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package cn.edu.pku.EOS.entity;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.UUID;
|
||||
|
||||
public class CrawlerTask {
|
||||
|
||||
public static final int WAITING = 0;
|
||||
public static final int SUCCESS = 1;
|
||||
public static final int ERROR = 2;
|
||||
|
||||
private String uuid = UUID.randomUUID().toString();
|
||||
private String projectUuid;
|
||||
private String crawlerNode;
|
||||
private String resourceType;
|
||||
private Date startTime = new Date();
|
||||
private Date finishTime = null;
|
||||
private int status = WAITING;
|
||||
|
||||
public CrawlerTask(){}
|
||||
|
||||
public CrawlerTask(Project project, String resoucetype) {
|
||||
this.projectUuid = project.getUuid();
|
||||
this.resourceType = resoucetype;
|
||||
}
|
||||
public String getUuid() {
|
||||
return uuid;
|
||||
}
|
||||
public void setUuid(String uuid) {
|
||||
this.uuid = uuid;
|
||||
}
|
||||
public String getProjectUuid() {
|
||||
return projectUuid;
|
||||
}
|
||||
public void setProjectUuid(String projectUuid) {
|
||||
this.projectUuid = projectUuid;
|
||||
}
|
||||
public String getCrawlerNode() {
|
||||
return crawlerNode;
|
||||
}
|
||||
public void setCrawlerNode(String crawlerNode) {
|
||||
this.crawlerNode = crawlerNode;
|
||||
}
|
||||
public String getResourceType() {
|
||||
return resourceType;
|
||||
}
|
||||
public void setResourceType(String resourceType) {
|
||||
this.resourceType = resourceType;
|
||||
}
|
||||
public Date getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
public void setStartTime(Date startTime) {
|
||||
this.startTime = startTime;
|
||||
}
|
||||
public Date getFinishTime() {
|
||||
return finishTime;
|
||||
}
|
||||
public void setFinishTime(Date finishTime) {
|
||||
this.finishTime = finishTime;
|
||||
}
|
||||
public int getStatus() {
|
||||
return status;
|
||||
}
|
||||
public void setStatus(int status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
package cn.edu.pku.EOS.entity;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import cn.edu.pku.EOS.util.XMLUtils;
|
||||
/**
|
||||
* 存储开源项目基本信息的类
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
|
||||
public class Project {
|
||||
|
||||
public static final String JAVA = "Java";
|
||||
public static final String CPP = "C++";
|
||||
public static final String C = "C";
|
||||
private String name;
|
||||
private String hostUrl;
|
||||
private String programmingLanguage = JAVA;
|
||||
private String description;
|
||||
private String uuid = UUID.randomUUID().toString();;
|
||||
|
||||
private List<ResourceMetaData> resources = new ArrayList<ResourceMetaData>();
|
||||
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 请用该构造方法得到uuid
|
||||
*/
|
||||
public Project(String _name, String _hostUrl) {
|
||||
name = _name;
|
||||
hostUrl = _hostUrl;
|
||||
uuid = UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 用该方法从xml文件得到新项目，包含uuid
|
||||
* @param xmlFile xml文件路径
|
||||
*/
|
||||
public Project(String xmlFile) {
|
||||
Project project;
|
||||
project = XMLUtils.getProjectFromXmlFile(xmlFile);
|
||||
name = project.getName();
|
||||
hostUrl = project.getHostUrl();
|
||||
programmingLanguage = project.getProgrammingLanguage();
|
||||
description = project.getDescription();
|
||||
resources = project.getResources();
|
||||
}
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 供javabean使用，不要手工调用该构造方法，否则无法得到uuid
|
||||
*/
|
||||
public Project() {}
|
||||
|
||||
public void addResourceMetaData(String type, String urls, String crawler) {
|
||||
resources.add(new ResourceMetaData(type, urls, crawler));
|
||||
}
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 根据资源类型得到该类型的资源metadata
|
||||
* @param resoucetype
|
||||
* @return
|
||||
*/
|
||||
public ResourceMetaData getResourceByType(String resoucetype) {
|
||||
if (resoucetype.equals(ResourceMetaData.RELATIVEWEB_TYPE)) {
|
||||
return new ResourceMetaData(resoucetype, null, "RelativeWebCrawlerVer2");
|
||||
}
|
||||
for (ResourceMetaData data : resources) {
|
||||
if (data.getType().equals(resoucetype)) {
|
||||
return data;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getUuid() {
|
||||
return uuid;
|
||||
}
|
||||
public void setUuid(String uuid) {
|
||||
this.uuid = uuid;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public String getHostUrl() {
|
||||
return hostUrl;
|
||||
}
|
||||
public void setHostUrl(String hostUrl) {
|
||||
this.hostUrl = hostUrl;
|
||||
}
|
||||
public String getProgrammingLanguage() {
|
||||
return programmingLanguage;
|
||||
}
|
||||
public void setProgrammingLanguage(String programmingLanguage) {
|
||||
this.programmingLanguage = programmingLanguage;
|
||||
}
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
public List<ResourceMetaData> getResources() {
|
||||
return resources;
|
||||
}
|
||||
public void setResources(List<ResourceMetaData> resources) {
|
||||
this.resources = resources;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
package cn.edu.pku.EOS.entity;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 存储开源项目和资源类型关系的类，对印的数据库表中存储了开源项目对因的资源类型，以及每种资源的爬取地址，以及爬虫类型等元信息
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
|
||||
public class ResourceMetaData {
|
||||
|
||||
public static final String DOC_TYPE = "Documentation";
|
||||
public static final String CODE_TYPE = "SourceCode";
|
||||
public static final String MAIL_TYPE = "MailingList";
|
||||
public static final String BUG_TYPE = "IssueTracker";
|
||||
public static final String QA_TYPE = "FAQ";
|
||||
public static final String SEARCH_TYPE = "Search";
|
||||
public static final String RELATIVEWEB_TYPE = "RelativeWeb";
|
||||
|
||||
public static final String[] RESOURCE_TYPE_LIST = {DOC_TYPE, CODE_TYPE, MAIL_TYPE, BUG_TYPE};
|
||||
|
||||
private String type;
|
||||
private String urlListString;
|
||||
private List<String> baseUrls = new ArrayList<String>();
|
||||
private String crawler;
|
||||
|
||||
|
||||
|
||||
public ResourceMetaData(){}
|
||||
public ResourceMetaData(String type, String urls, String crawler) {
|
||||
this.type = type;
|
||||
this.urlListString = urls;
|
||||
this.crawler = crawler;
|
||||
getUrlsFromUrlString(urls);
|
||||
}
|
||||
|
||||
public String getUrlListString() {
|
||||
return urlListString;
|
||||
}
|
||||
public void setUrlListString(String urlListsString) {
|
||||
this.urlListString = urlListsString;
|
||||
}
|
||||
public String getCrawler() {
|
||||
return crawler;
|
||||
}
|
||||
public void setCrawler(String crawler) {
|
||||
this.crawler = crawler;
|
||||
}
|
||||
public List<String> getBaseUrls() {
|
||||
return baseUrls;
|
||||
}
|
||||
public void setBaseUrls(List<String> baseUrls) {
|
||||
this.baseUrls = baseUrls;
|
||||
urlListString = baseUrls.get(0);
|
||||
for (int i = 1; i < baseUrls.size(); i++) {
|
||||
urlListString = urlListString + ";" + baseUrls.get(i);
|
||||
}
|
||||
}
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
/**
|
||||
* @author 张灵箫
|
||||
* 数据库中urlliststring存了多个url地址，用分号;隔开，该方法将这些地址存入baseUrls中
|
||||
* 注意：会清空原来的baseurl中的内容！
|
||||
* @param urlListsString
|
||||
*/
|
||||
public void getUrlsFromUrlString(String urlListsString) {
|
||||
if (urlListsString == null) {
|
||||
return;
|
||||
}
|
||||
String[] urlList = urlListsString.split(";");
|
||||
baseUrls.clear();
|
||||
for (String string : urlList) {
|
||||
baseUrls.add(string);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
ResourceMetaData rData = new ResourceMetaData();
|
||||
rData.getUrlsFromUrlString("http://lucene.apache.org/core/4_3_0/index.html;http://lucene.apache.org/core/3_6_2/index.html");
|
||||
for (String string : rData.getBaseUrls()) {
|
||||
System.out.println(string);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package cn.edu.pku.EOS.exception;
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
public class CrawlerNodeNotRespondingException extends Exception {
|
||||
|
||||
private String host;
|
||||
public CrawlerNodeNotRespondingException(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage(){
|
||||
return "Can't not connect to host " + host;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package cn.edu.pku.EOS.exception;
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
|
||||
public class CrawlerNotInNodeListException extends Exception {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package cn.edu.pku.EOS.nodedispatch;
|
||||
|
||||
/**
|
||||
* 表示一个爬虫节点的类，包括节点IP地址和节点的爬虫类型
|
||||
* 爬虫和IP地址应该是多对多的关系
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class CrawlerNode {
|
||||
private String crawlerType;
|
||||
private String hostAddress;
|
||||
public CrawlerNode(String ip, String crawlerClassName) {
|
||||
crawlerType = crawlerClassName;
|
||||
hostAddress = ip;
|
||||
}
|
||||
public String getCrawlerType() {
|
||||
return crawlerType;
|
||||
}
|
||||
public void setCrawlerType(String crawlerType) {
|
||||
this.crawlerType = crawlerType;
|
||||
}
|
||||
public String getHostAddress() {
|
||||
return hostAddress;
|
||||
}
|
||||
public void setHostAddress(String hostAddress) {
|
||||
this.hostAddress = hostAddress;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package cn.edu.pku.EOS.nodedispatch;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
import cn.edu.pku.EOS.entity.CrawlerTask;
|
||||
import cn.edu.pku.EOS.exception.CrawlerNodeNotRespondingException;
|
||||
import cn.edu.pku.EOS.exception.CrawlerNotInNodeListException;
|
||||
|
||||
/**
|
||||
* 把爬取任务分配给不同的节点
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class JobDispatcher {
|
||||
|
||||
List<CrawlerNode> nodes = null;
|
||||
|
||||
public JobDispatcher(String crawlerClassName) throws CrawlerNotInNodeListException {
|
||||
nodes = new ArrayList<CrawlerNode>();
|
||||
List<String> ipList = NodeManager.getNodeList(crawlerClassName);
|
||||
if (ipList == null) {
|
||||
return;
|
||||
}
|
||||
for (String ip : ipList) {
|
||||
nodes.add(new CrawlerNode(ip, crawlerClassName));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 以随机方式分配任务到某个节点
|
||||
* @author 张灵箫
|
||||
* @param resoucetype
|
||||
* @param project
|
||||
* @return
|
||||
* @throws CrawlerNodeNotRespondingException
|
||||
* @throws CrawlerNotInNodeListException
|
||||
*/
|
||||
public String dispatchCrawlerJob(CrawlerTask crawlerTask) throws CrawlerNodeNotRespondingException {
|
||||
int nodenum = nodes.size();
|
||||
int chosen = (int) (Math.random() * nodenum);
|
||||
String host = nodes.get(chosen).getHostAddress();
|
||||
crawlerTask.setCrawlerNode(host);
|
||||
String commandName = NodeManager.getCrawlCommand();
|
||||
List<KVPair> params = new ArrayList<KVPair>();
|
||||
params.add(new KVPair("type", crawlerTask.getResourceType()));
|
||||
params.add(new KVPair("projectuuid", crawlerTask.getProjectUuid()));
|
||||
params.add(new KVPair("taskuuid", crawlerTask.getUuid()));
|
||||
return MessageSender.sendCommand(host, commandName, params);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package cn.edu.pku.EOS.nodedispatch;
|
||||
|
||||
public class KVPair {
|
||||
private String key;
|
||||
private String value;
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
public void setKey(String key) {
|
||||
this.key = key;
|
||||
}
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public KVPair(String k, String v) {
|
||||
key = k;
|
||||
value = v;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
package cn.edu.pku.EOS.nodedispatch;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
import org.apache.commons.httpclient.HttpException;
|
||||
import org.apache.commons.httpclient.HttpStatus;
|
||||
import org.apache.commons.httpclient.methods.PostMethod;
|
||||
import org.apache.commons.httpclient.params.HttpMethodParams;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.eclipse.jdt.internal.compiler.ast.ThisReference;
|
||||
|
||||
import cn.edu.pku.EOS.exception.CrawlerNodeNotRespondingException;
|
||||
|
||||
|
||||
/**
|
||||
* 实现节点间的底层通信协议
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class MessageSender {
|
||||
|
||||
public static final String REMOTE_SUCCESS_RESPONCE = "success";
|
||||
private static final Logger logger = Logger.getLogger(ThisReference.class.getName());
|
||||
/**
|
||||
* 发送http请求到host
|
||||
* @author 张灵箫
|
||||
* @param host
|
||||
* @param commandName
|
||||
* @param params
|
||||
* @return
|
||||
* @throws CrawlerNodeNotRespondingException
|
||||
*/
|
||||
public static String sendCommand(String host, String commandName,
|
||||
List<KVPair> params) throws CrawlerNodeNotRespondingException {
|
||||
String result = null;
|
||||
|
||||
//得到http请求的url格式，含参数
|
||||
String httpRequestUrl = "http://" + host + "/" + NodeManager.getNodeServiceName() + "/" + commandName + "?";
|
||||
for (KVPair kvPair : params) {
|
||||
httpRequestUrl += kvPair.getKey() + "=" +kvPair.getValue() + "&";
|
||||
}
|
||||
httpRequestUrl = httpRequestUrl.substring(0, httpRequestUrl.length()-1);
|
||||
logger.info("Requesting url: " +httpRequestUrl);
|
||||
|
||||
HttpClient client = new HttpClient();
|
||||
PostMethod post = new PostMethod(httpRequestUrl);
|
||||
post.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
|
||||
new DefaultHttpMethodRetryHandler(0, false));
|
||||
|
||||
try {
|
||||
// Execute the method.
|
||||
int statusCode = client.executeMethod(post);
|
||||
if (statusCode != HttpStatus.SC_OK) {
|
||||
System.err.println("Method failed: " + post.getStatusLine());
|
||||
}
|
||||
byte[] responseBody = post.getResponseBody();
|
||||
result = new String(responseBody);
|
||||
} catch (HttpException e) {
|
||||
System.err.println("Fatal protocol violation: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
throw new CrawlerNodeNotRespondingException(host);
|
||||
} catch (IOException e) {
|
||||
System.err.println("Fatal transport error: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
throw new CrawlerNodeNotRespondingException(host);
|
||||
} finally {
|
||||
// Release the connection.
|
||||
post.releaseConnection();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package cn.edu.pku.EOS.nodedispatch;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
import cn.edu.pku.EOS.exception.CrawlerNotInNodeListException;
|
||||
/**
|
||||
*
|
||||
* @author 灵箫
|
||||
* 从配置文件爬虫节点
|
||||
*/
|
||||
public class NodeManager {
|
||||
// 属性文件标识符
|
||||
private static String NODELIST_FILE_NAME = "cn.edu.pku.EOS.nodedispatch.crawlerNodes";
|
||||
|
||||
// 所使用的ResourceBundle
|
||||
private static ResourceBundle bundle;
|
||||
|
||||
// 静态私有方法，用于从属性文件中取得属性值
|
||||
static {
|
||||
try {
|
||||
//System.out.println("init config file!");
|
||||
bundle = ResourceBundle.getBundle(NODELIST_FILE_NAME);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String getValue(String key) {
|
||||
try {
|
||||
return bundle.getString(key);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static List<String> getNodeList(String crawlerClass) throws CrawlerNotInNodeListException {
|
||||
List<String> nodeList = new ArrayList<String>();
|
||||
if (getValue(crawlerClass) == null) {
|
||||
throw new CrawlerNotInNodeListException();
|
||||
}
|
||||
String[] nodes = getValue(crawlerClass).split(";");
|
||||
for (String string : nodes) {
|
||||
nodeList.add(string);
|
||||
}
|
||||
return nodeList;
|
||||
}
|
||||
|
||||
public static String getNodeServiceName() {
|
||||
return getValue("servicename");
|
||||
}
|
||||
|
||||
public static String getCheckTaskCommand() {
|
||||
return getValue("checktaskcommand");
|
||||
}
|
||||
|
||||
public static String getCrawlCommand() {
|
||||
return getValue("crawlcommand");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// //List<String> node = getNodeList("TestCrawler");
|
||||
// for (String string : node) {
|
||||
// System.out.println(string);
|
||||
// }
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
servicename=EOSCrawlerNode
|
||||
#cmmand list
|
||||
crawlcommand=CrawlResource
|
||||
checktaskcommand=IsTaskAlive
|
||||
#IP list
|
||||
TestCrawler=192.168.2.221:8080;192.168.4.204:8080
|
||||
HtmlDocCrawler=192.168.4.204:8080
|
||||
ApacheMboxCrawler=192.168.4.204:8080
|
||||
SVNCrawler=192.168.4.204:8080
|
||||
JiraCrawler=192.168.4.204:8080
|
||||
RelativeWebCrawler=192.168.4.204:8080
|
||||
RelativeWebCrawlerVer2=192.168.4.204:8080
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
package cn.edu.pku.EOS.profileGenerator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.httpclient.HttpException;
|
||||
import org.htmlparser.Node;
|
||||
import org.htmlparser.Parser;
|
||||
import org.htmlparser.filters.StringFilter;
|
||||
import org.htmlparser.filters.TagNameFilter;
|
||||
import org.htmlparser.nodes.TagNode;
|
||||
import org.htmlparser.util.NodeList;
|
||||
import org.htmlparser.util.ParserException;
|
||||
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
|
||||
public class ApacheProjectParser extends ProjectMetaDataParser {
|
||||
|
||||
@Override
|
||||
protected String parseProgrammingLanguage(String htmlPageString) throws ParserException {
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("Programming Languages"));
|
||||
return list.elementAt(1).getParent().getNextSibling().getNextSibling().toPlainTextString().trim();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String parseHostUrl(String htmlPageString) throws ParserException {
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("Project Website"));
|
||||
return list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String parseDescription(String htmlPageString) throws ParserException {
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new TagNameFilter("p"));
|
||||
return list.elementAt(1).toPlainTextString().trim();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String parseName(String htmlPageString) throws ParserException {
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new TagNameFilter("h2"));
|
||||
return list.elementAt(0).toPlainTextString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ResourceMetaData parseCodeMetaData(String htmlPageString) throws ParserException {
|
||||
ResourceMetaData data = new ResourceMetaData();
|
||||
data.setType(ResourceMetaData.CODE_TYPE);
|
||||
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("SVN Direct"));
|
||||
if (list.size() == 0) {
|
||||
return data;
|
||||
}
|
||||
urlStrings.add(list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim());
|
||||
data.setBaseUrls(urlStrings);
|
||||
|
||||
data.setCrawler("SVNCrawler");
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ResourceMetaData parseBugMetaData(String htmlPageString) throws ParserException {
|
||||
ResourceMetaData data = new ResourceMetaData();
|
||||
data.setType(ResourceMetaData.BUG_TYPE);
|
||||
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("Bug/Issue Tracker"));
|
||||
if (list.size() == 0) {
|
||||
return data;
|
||||
}
|
||||
|
||||
String url = list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim();
|
||||
|
||||
if (url.contains("jira")) {
|
||||
if (!url.contains("https")) {
|
||||
url = "https" + url.substring(4, url.length());
|
||||
}
|
||||
}
|
||||
|
||||
urlStrings.add(url);
|
||||
data.setBaseUrls(urlStrings);
|
||||
|
||||
if(urlStrings.get(0).contains("jira")) {
|
||||
data.setCrawler("JiraCrawler");
|
||||
} else if (urlStrings.get(0).contains("bugzilla")) {
|
||||
data.setCrawler("BugzillaCrawler");
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ResourceMetaData parseMailingListMetaData(String htmlPageString) throws ParserException {
|
||||
ResourceMetaData data = new ResourceMetaData();
|
||||
data.setType(ResourceMetaData.MAIL_TYPE);
|
||||
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("Mailing Lists"));
|
||||
if (list.size() == 0) {
|
||||
return data;
|
||||
}
|
||||
String currentUrlString = list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim();
|
||||
if (currentUrlString.contains("http://mail-archives.apache.org/mod_mbox/")) {
|
||||
urlStrings.add(currentUrlString);
|
||||
} else {
|
||||
String newHtmlString;
|
||||
try {
|
||||
newHtmlString = getHtmlStringFromUrl(list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim());
|
||||
} catch (HttpException e) {
|
||||
e.printStackTrace();
|
||||
return data;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return data;
|
||||
}
|
||||
urlStrings = parseArchiveUrlFromHTML(newHtmlString);
|
||||
if (urlStrings == null) {
|
||||
urlStrings = findMailArchiveFromApacheList();
|
||||
}
|
||||
}
|
||||
|
||||
if (urlStrings == null) {
|
||||
return data;
|
||||
}
|
||||
data.setBaseUrls(urlStrings);
|
||||
|
||||
data.setCrawler("ApacheMboxCrawler");
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private List<String> findMailArchiveFromApacheList() throws ParserException {
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
String projectName = pageUrlString.substring(36);
|
||||
projectName = projectName.substring(0, projectName.length() - 5);
|
||||
//System.out.println(projectName);
|
||||
|
||||
Parser parser = new Parser();
|
||||
try {
|
||||
parser.setInputHTML(getHtmlStringFromUrl("http://mail-archives.apache.org/mod_mbox/"));
|
||||
} catch (HttpException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter(projectName+"."));
|
||||
Node[] nodes = list.elementAt(0).getParent().getParent().getNextSibling().getNextSibling().getChildren().toNodeArray();
|
||||
for (Node node : nodes) {
|
||||
if (node.toHtml().contains("user")) {
|
||||
TagNode tagNode = (TagNode) node.getFirstChild();
|
||||
urlStrings.add("http://mail-archives.apache.org/mod_mbox/" + tagNode.getAttribute("href"));
|
||||
}
|
||||
}
|
||||
return urlStrings;
|
||||
}
|
||||
|
||||
private List<String> parseArchiveUrlFromHTML(String newHtmlString) {
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
|
||||
Parser parser = new Parser();
|
||||
try {
|
||||
parser.setInputHTML(newHtmlString);
|
||||
Node[] nodes = parser.extractAllNodesThatMatch(new TagNameFilter("a")).toNodeArray();
|
||||
for (Node node : nodes) {
|
||||
String urlString = ((TagNode) node).getAttribute("href");
|
||||
if (urlString != null) {
|
||||
if (urlString.contains("http://mail-archives.apache.org/mod_mbox/")
|
||||
&& urlString.contains("user")) {
|
||||
urlStrings.add(urlString);
|
||||
return urlStrings;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (ParserException e) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ResourceMetaData parseDocumentationMetaData(String htmlPageString) throws ParserException {
|
||||
ResourceMetaData data = new ResourceMetaData();
|
||||
data.setType(ResourceMetaData.DOC_TYPE);
|
||||
|
||||
List<String> urlStrings = new ArrayList<String>();
|
||||
Parser parser = new Parser();
|
||||
parser.setInputHTML(htmlPageString);
|
||||
NodeList list = parser.extractAllNodesThatMatch(new StringFilter("Project Website"));
|
||||
urlStrings.add(list.elementAt(0).getParent().getNextSibling().getNextSibling().toPlainTextString().trim());
|
||||
data.setBaseUrls(urlStrings);
|
||||
|
||||
data.setCrawler("HtmlDocCrawler");
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
package cn.edu.pku.EOS.profileGenerator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
import org.apache.commons.httpclient.HttpException;
|
||||
import org.apache.commons.httpclient.HttpStatus;
|
||||
import org.apache.commons.httpclient.methods.GetMethod;
|
||||
import org.apache.commons.httpclient.params.HttpMethodParams;
|
||||
import org.htmlparser.util.ParserException;
|
||||
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
|
||||
public abstract class ProjectMetaDataParser {
|
||||
|
||||
String pageUrlString = null;
|
||||
|
||||
public Project parseProjectInfoFromWebPage(String pageurl) throws IOException {
|
||||
pageUrlString = pageurl;
|
||||
Project project = new Project();
|
||||
String htmlPageString = getHtmlStringFromUrl(pageurl);
|
||||
|
||||
try {
|
||||
project.setName(parseName(htmlPageString));
|
||||
project.setDescription(parseDescription(htmlPageString));
|
||||
project.setHostUrl(parseHostUrl(htmlPageString));
|
||||
project.setProgrammingLanguage(parseProgrammingLanguage(htmlPageString));
|
||||
|
||||
List<ResourceMetaData> datas = new ArrayList<ResourceMetaData>();
|
||||
for (String resourceType : ResourceMetaData.RESOURCE_TYPE_LIST) {
|
||||
ResourceMetaData data = parseResourceMetaData(resourceType, htmlPageString);
|
||||
if (data != null) {
|
||||
datas.add(data);
|
||||
}
|
||||
}
|
||||
project.setResources(datas);
|
||||
} catch (ParserException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
|
||||
return project;
|
||||
}
|
||||
|
||||
private ResourceMetaData parseResourceMetaData(String resourceType, String htmlPageString) throws ParserException {
|
||||
if (resourceType.equals(ResourceMetaData.DOC_TYPE)) {
|
||||
return parseDocumentationMetaData(htmlPageString);
|
||||
}
|
||||
if (resourceType.equals(ResourceMetaData.MAIL_TYPE)) {
|
||||
return parseMailingListMetaData(htmlPageString);
|
||||
}
|
||||
if (resourceType.equals(ResourceMetaData.BUG_TYPE)) {
|
||||
return parseBugMetaData(htmlPageString);
|
||||
}
|
||||
if (resourceType.equals(ResourceMetaData.CODE_TYPE)) {
|
||||
return parseCodeMetaData(htmlPageString);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
protected abstract ResourceMetaData parseCodeMetaData(String htmlPageString) throws ParserException;
|
||||
protected abstract ResourceMetaData parseBugMetaData(String htmlPageString) throws ParserException;
|
||||
protected abstract ResourceMetaData parseMailingListMetaData(String htmlPageString) throws ParserException;
|
||||
protected abstract ResourceMetaData parseDocumentationMetaData(String htmlPageString) throws ParserException;
|
||||
|
||||
protected abstract String parseProgrammingLanguage(String htmlPageString)throws ParserException;
|
||||
protected abstract String parseHostUrl(String htmlPageString)throws ParserException;
|
||||
protected abstract String parseDescription(String htmlPageString)throws ParserException;
|
||||
protected abstract String parseName(String htmlPageString)throws ParserException;
|
||||
|
||||
protected String getHtmlStringFromUrl(String pageurl) throws IOException,
|
||||
HttpException {
|
||||
HttpClient client = new HttpClient();
|
||||
GetMethod get = new GetMethod(pageurl);
|
||||
get.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
|
||||
new DefaultHttpMethodRetryHandler(0, false));
|
||||
|
||||
int statusCode = client.executeMethod(get);
|
||||
if (statusCode != HttpStatus.SC_OK) {
|
||||
System.err.println("Method failed: " + get.getStatusLine());
|
||||
}
|
||||
byte[] responseBody = get.getResponseBody();
|
||||
String htmlPageString = new String(responseBody);
|
||||
|
||||
//System.out.println(htmlPageString);
|
||||
return htmlPageString;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package cn.edu.pku.EOS.profileGenerator;
|
||||
|
||||
public class ProjectMetaDataParserFactory {
|
||||
public static final String APACHE_TYPE = "apache";
|
||||
public static final String UNKNOWN_TYPE = "unknown";
|
||||
|
||||
|
||||
public static String inferParserTypeFromUrl(String pageurl) {
|
||||
if (pageurl.contains("apache")) {
|
||||
return APACHE_TYPE;
|
||||
}
|
||||
return UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
public static ProjectMetaDataParser getParser(String type) {
|
||||
if (type.equals(APACHE_TYPE)) {
|
||||
return new ApacheProjectParser();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package cn.edu.pku.EOS.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import cn.edu.pku.EOS.business.ProjectBusiness;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
import cn.edu.pku.EOS.entity.ResourceMetaData;
|
||||
|
||||
/**
|
||||
* Servlet implementation class CrawlAllResource
|
||||
*/
|
||||
public class CrawlAllResource extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* @see HttpServlet#HttpServlet()
|
||||
*/
|
||||
public CrawlAllResource() {
|
||||
super();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
String projectUuid = request.getParameter("projectuuid");
|
||||
ProjectBusiness pb = new ProjectBusiness();
|
||||
Project project = pb.getProjectByUuid(projectUuid);
|
||||
List<ResourceMetaData> datas = project.getResources();
|
||||
for (ResourceMetaData data : datas) {
|
||||
pb.startNewCrawlerTask(data.getType(), project);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package cn.edu.pku.EOS.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import cn.edu.pku.EOS.business.TaskBusiness;
|
||||
|
||||
/**
|
||||
* Servlet implementation class GetTaskStatus
|
||||
*/
|
||||
public class GetTaskStatus extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* @see HttpServlet#HttpServlet()
|
||||
*/
|
||||
public GetTaskStatus() {
|
||||
super();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
String uuid = request.getParameter("taskuuid");
|
||||
int status = TaskBusiness.checkTaskStatus(uuid);
|
||||
|
||||
response.setContentType("text/plain");
|
||||
response.getWriter().print(status);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package cn.edu.pku.EOS.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import cn.edu.pku.EOS.business.InitBusiness;
|
||||
|
||||
/**
|
||||
* 系统启动时调用的servlet，对系统进行初始化
|
||||
* @author 张灵箫
|
||||
*
|
||||
*/
|
||||
public class InitServlet extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* @see HttpServlet#HttpServlet()
|
||||
* tomcat启动时自动调用该方法
|
||||
* @author 张灵箫
|
||||
*/
|
||||
public InitServlet() {
|
||||
super();
|
||||
InitBusiness.initEOS();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package cn.edu.pku.EOS.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import cn.edu.pku.EOS.business.ProjectBusiness;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
|
||||
/**
|
||||
* Servlet implementation class CrawlResource
|
||||
*/
|
||||
public class StartCrawlerTask extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* @see HttpServlet#HttpServlet()
|
||||
*/
|
||||
public StartCrawlerTask() {
|
||||
super();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
String uuid = request.getParameter("projectuuid");
|
||||
String type = request.getParameter("type");
|
||||
ProjectBusiness pb = new ProjectBusiness();
|
||||
Project p = pb.getProjectByUuid(uuid);
|
||||
String result = pb.startNewCrawlerTask(type, p);
|
||||
|
||||
response.setContentType("text/plain");
|
||||
response.getWriter().print(result);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
package cn.edu.pku.EOS.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import cn.edu.pku.EOS.TestUtil;
|
||||
import cn.edu.pku.EOS.business.ProjectBusiness;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
|
||||
/**
|
||||
* Servlet implementation class Test
|
||||
* 用于测试的servlet
|
||||
* @author 张灵箫
|
||||
*/
|
||||
public class Test extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* @see HttpServlet#HttpServlet()
|
||||
*/
|
||||
public Test() {
|
||||
super();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
doPost(request, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
*/
|
||||
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
ProjectBusiness pb = new ProjectBusiness();
|
||||
String url = request.getParameter("url");
|
||||
Project project = pb.parseProjectInfoFromWebPage(url);
|
||||
Project project2 = pb.getProjectByName(project.getName());
|
||||
if (project2 != null) {
|
||||
pb.updateProjectInfo(project, project2.getUuid());
|
||||
TestUtil.printProjectInfo(project);
|
||||
System.out.println("Test Success!");
|
||||
return;
|
||||
}
|
||||
TestUtil.printProjectInfo(project);
|
||||
pb.createNewProject(project);
|
||||
System.out.println("Test Success!");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package cn.edu.pku.EOS.util;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import jcifs.smb.SmbFile;
|
||||
import jcifs.smb.SmbFileInputStream;
|
||||
import jcifs.smb.SmbFileOutputStream;
|
||||
|
||||
public class RemoteFileOperation {
|
||||
/**
|
||||
* @Title: smbGet
|
||||
|
||||
* @Description: 从一个共享地址下载一个文件
|
||||
|
||||
* @param @param remoteUrl 共享文件地址
|
||||
* @param @param localFilePath 本地地址
|
||||
|
||||
* @return void 返回类型
|
||||
|
||||
* @author: Huazb (huazb1989@126.com) Software Engineering Institute, Peking
|
||||
* University, China
|
||||
|
||||
* @throws
|
||||
@case RemoteFileOperation.smbGet("smb://192.168.4.9/Upload/Temp/test_2.txt","d:/");
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
public static String smbGet(String remoteUrl,String localDir) {
|
||||
InputStream in = null;
|
||||
OutputStream out = null;
|
||||
try {
|
||||
SmbFile remoteFile = new SmbFile(remoteUrl);
|
||||
if(remoteFile==null){
|
||||
System.out.println("共享文件不存在");
|
||||
return null;
|
||||
}
|
||||
String fileName = remoteFile.getName();
|
||||
File localFile = new File(localDir+File.separator+fileName);
|
||||
in = new BufferedInputStream(new SmbFileInputStream(remoteFile));
|
||||
out = new BufferedOutputStream(new FileOutputStream(localFile));
|
||||
byte[] buffer = new byte[1024];
|
||||
int len;
|
||||
while(true){
|
||||
len=in.read(buffer);
|
||||
// System.out.println("len="+len);
|
||||
if (len==-1) break;
|
||||
out.write(buffer, 0, len);
|
||||
|
||||
//out.write(buffer);
|
||||
buffer = new byte[1024];
|
||||
}
|
||||
return localDir+File.separator+fileName;
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
in.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @Title: smbPut
|
||||
|
||||
* @Description: 向一个共享地址上传一个文件
|
||||
|
||||
* @param @param remoteUrl 共享地址
|
||||
* @param @param localFilePath 本地文件地址
|
||||
|
||||
* @return void 返回类型
|
||||
|
||||
* @author: Huazb (huazb1989@126.com) Software Engineering Institute, Peking
|
||||
* University, China
|
||||
|
||||
* @throws
|
||||
@case RemoteFileOperation.smbPut("smb://192.168.4.9/Upload/Temp", "d:/test_1.txt");
|
||||
*/
|
||||
public static void smbPut(String remoteUrl,String localFilePath) {
|
||||
InputStream in = null;
|
||||
OutputStream out = null;
|
||||
try {
|
||||
File localFile = new File(localFilePath);
|
||||
|
||||
String fileName = localFile.getName();
|
||||
SmbFile remoteFile = new SmbFile(remoteUrl+"/"+fileName);
|
||||
in = new BufferedInputStream(new FileInputStream(localFile));
|
||||
out = new BufferedOutputStream(new SmbFileOutputStream(remoteFile));
|
||||
byte[] buffer = new byte[1024];
|
||||
int len;
|
||||
while(true){
|
||||
len=in.read(buffer);
|
||||
// System.out.println("len="+len);
|
||||
if (len==-1) break;
|
||||
out.write(buffer, 0, len);
|
||||
//out.write(buffer);
|
||||
buffer = new byte[1024];
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
in.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
public static void main(String args[]) {
|
||||
/*
|
||||
* 远程url smb://192.168.0.77/test如果需要用户名密码就这样：
|
||||
smb://username:password@192.168.0.77/test
|
||||
|
||||
*/
|
||||
// RemoteFileOperation.smbPut("smb://192.168.4.9/Upload/Temp", "d:/test_1.txt");
|
||||
RemoteFileOperation.smbGet("smb://192.168.4.9/Upload/Temp/test_2.txt","d:/");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package cn.edu.pku.EOS.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.commons.digester3.Digester;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import cn.edu.pku.EOS.TestUtil;
|
||||
import cn.edu.pku.EOS.config.Config;
|
||||
import cn.edu.pku.EOS.entity.Project;
|
||||
|
||||
public class XMLUtils {
|
||||
public static Project getProjectFromXmlFile(String remotefilepath) {
|
||||
if (remotefilepath.startsWith("smb")) {
|
||||
remotefilepath = RemoteFileOperation.smbGet(remotefilepath, Config.getTempDir());
|
||||
}
|
||||
|
||||
|
||||
Project project = null;
|
||||
File xmlFile = new File(remotefilepath);
|
||||
Digester digester = new Digester();
|
||||
digester.addObjectCreate("project", Project.class);
|
||||
digester.addCallMethod("project/project_info/name", "setName", 0);
|
||||
digester.addCallMethod("project/project_info/hostUrl", "setHostUrl", 0);
|
||||
digester.addCallMethod("project/project_info/programmingLanguage", "setProgrammingLanguage", 0);
|
||||
digester.addCallMethod("project/project_info/description", "setDescription", 0);
|
||||
digester.addCallMethod("project/resources/resource", "addResourceMetaData", 3);
|
||||
digester.addCallParam("project/resources/resource/type", 0);
|
||||
digester.addCallParam("project/resources/resource/urls", 1);
|
||||
digester.addCallParam("project/resources/resource/crawler", 2);
|
||||
try {
|
||||
project = digester.parse(xmlFile);
|
||||
} catch (IOException e) {
|
||||
System.out.println("XML file not exist!");
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
} catch (SAXException e) {
|
||||
System.out.println("XML file schema incorrect!");
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Project project = null;
|
||||
project = getProjectFromXmlFile("D:/EOSdir/b7914db3-caa7-4d70-96cd-bd4b5b4ed029/b7914db3-caa7-4d70-96cd-bd4b5b4ed029.xml");
|
||||
TestUtil.printProjectInfo(project);
|
||||
}
|
||||
|
||||
public static String getProjectXmlFile(String uuid) {
|
||||
return Config.getEOSDir() + "/" + uuid + "/" + uuid + ".xml";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
log4j.rootLogger=INFO,CONSOLE,FILE
|
||||
log4j.addivity.org.apache=true
|
||||
#For console
|
||||
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.Threshold=ERROR
|
||||
log4j.appender.CONSOLE.Target=System.out
|
||||
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.CONSOLE.layout.ConversionPattern=[EOS]%-5p %d - %m%n
|
||||
#For file
|
||||
log4j.appender.FILE=org.apache.log4j.FileAppender
|
||||
log4j.appender.FILE.File=log/file.log
|
||||
log4j.appender.FILE.Append=false
|
||||
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.FILE.layout.ConversionPattern=[EOS]%-5p %d - %m%n
|
||||
Loading…
Reference in New Issue