0
|
1 <?xml version="1.0"?>
|
|
2 <!--
|
|
3 Licensed to the Apache Software Foundation (ASF) under one or more
|
|
4 contributor license agreements. See the NOTICE file distributed with
|
|
5 this work for additional information regarding copyright ownership.
|
|
6 The ASF licenses this file to You under the Apache License, Version 2.0
|
|
7 (the "License"); you may not use this file except in compliance with
|
|
8 the License. You may obtain a copy of the License at
|
|
9
|
|
10 http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
|
12 Unless required by applicable law or agreed to in writing, software
|
|
13 distributed under the License is distributed on an "AS IS" BASIS,
|
|
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15 See the License for the specific language governing permissions and
|
|
16 limitations under the License.
|
|
17 -->
|
|
18 <project name="${name}" default="runtime" xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:artifact="antlib:org.apache.maven.artifact.ant">
|
|
19
|
|
20 <!-- Load all the default properties, and any the user wants -->
|
|
21 <!-- to contribute (without having to type -D or edit this file -->
|
|
22 <property file="${user.home}/build.properties" />
|
|
23 <property file="${basedir}/build.properties" />
|
|
24 <property file="${basedir}/default.properties" />
|
|
25 <property name="test.junit.output.format" value="plain"/>
|
|
26 <property name="release.dir" value="${build.dir}/release"/>
|
|
27
|
|
28 <!-- define Maven coordinates, repository url and artifacts name etc -->
|
|
29 <property name="groupId" value="org.apache.nutch" />
|
|
30 <property name="artifactId" value="nutch" />
|
|
31 <property name="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2" />
|
|
32 <property name="maven-repository-id" value="apache.releases.https" />
|
|
33 <property name="maven-jar" value="${release.dir}/${artifactId}-${version}.jar" />
|
|
34 <property name="maven-javadoc-jar" value="${release.dir}/${artifactId}-${version}-javadoc.jar" />
|
|
35 <property name="maven-sources-jar" value="${release.dir}/${artifactId}-${version}-sources.jar" />
|
|
36
|
|
37
|
|
38 <!-- the normal classpath -->
|
|
39 <path id="classpath">
|
|
40 <pathelement location="${build.classes}"/>
|
|
41 <fileset dir="${build.lib.dir}">
|
|
42 <include name="*.jar" />
|
|
43 </fileset>
|
|
44 </path>
|
|
45
|
|
46 <!-- the unit test classpath -->
|
|
47 <dirname property="plugins.classpath.dir" file="${build.plugins}"/>
|
|
48 <path id="test.classpath">
|
|
49 <pathelement location="${test.build.classes}" />
|
|
50 <pathelement location="${conf.dir}"/>
|
|
51 <pathelement location="${test.src.dir}"/>
|
|
52 <pathelement location="${plugins.classpath.dir}"/>
|
|
53 <path refid="classpath"/>
|
|
54 <pathelement location="${build.dir}/${final.name}.job" />
|
|
55 <fileset dir="${build.lib.dir}">
|
|
56 <include name="*.jar" />
|
|
57 </fileset>
|
|
58 </path>
|
|
59
|
|
60 <!-- ====================================================== -->
|
|
61 <!-- Stuff needed by all targets -->
|
|
62 <!-- ====================================================== -->
|
|
63 <target name="init" depends="ivy-init" description="--> stuff required by all targets">
|
|
64 <mkdir dir="${build.dir}"/>
|
|
65 <mkdir dir="${build.classes}"/>
|
|
66 <mkdir dir="${release.dir}"/>
|
|
67
|
|
68 <mkdir dir="${test.build.dir}"/>
|
|
69 <mkdir dir="${test.build.classes}"/>
|
|
70
|
|
71 <touch datetime="01/25/1971 2:00 pm">
|
|
72 <fileset dir="${conf.dir}" includes="**/*.template"/>
|
|
73 </touch>
|
|
74
|
|
75 <copy todir="${conf.dir}" verbose="true">
|
|
76 <fileset dir="${conf.dir}" includes="**/*.template"/>
|
|
77 <mapper type="glob" from="*.template" to="*"/>
|
|
78 </copy>
|
|
79 </target>
|
|
80
|
|
81 <!-- ====================================================== -->
|
|
82 <!-- Compile the Java files -->
|
|
83 <!-- ====================================================== -->
|
|
84 <target name="compile" depends="compile-core, compile-plugins" description="--> compile all Java files"/>
|
|
85
|
|
86 <target name="compile-core" depends="init, resolve-default" description="--> compile core Java files only">
|
|
87 <javac
|
|
88 encoding="${build.encoding}"
|
|
89 srcdir="${src.dir}"
|
|
90 includes="org/apache/nutch/**/*.java"
|
|
91 destdir="${build.classes}"
|
|
92 debug="${javac.debug}"
|
|
93 optimize="${javac.optimize}"
|
|
94 target="${javac.version}"
|
|
95 source="${javac.version}"
|
|
96 deprecation="${javac.deprecation}">
|
|
97 <compilerarg value="-Xlint"/>
|
|
98 <classpath refid="classpath"/>
|
|
99 </javac>
|
|
100 </target>
|
|
101
|
|
102 <target name="compile-plugins" depends="init, resolve-default" description="--> compile plugins only">
|
|
103 <ant dir="src/plugin" target="deploy" inheritAll="false"/>
|
|
104 </target>
|
|
105
|
|
106 <!-- ================================================================== -->
|
|
107 <!-- Make nutch.jar -->
|
|
108 <!-- ================================================================== -->
|
|
109 <!-- -->
|
|
110 <!-- ================================================================== -->
|
|
111 <target name="jar" depends="compile-core" description="--> make nutch.jar">
|
|
112 <copy file="${conf.dir}/nutch-default.xml"
|
|
113 todir="${build.classes}"/>
|
|
114 <copy file="${conf.dir}/nutch-site.xml"
|
|
115 todir="${build.classes}"/>
|
|
116 <jar jarfile="${build.dir}/${final.name}.jar"
|
|
117 basedir="${build.classes}">
|
|
118 <manifest>
|
|
119 </manifest>
|
|
120 </jar>
|
|
121 </target>
|
|
122
|
|
123 <!-- ================================================================== -->
|
|
124 <!-- Make Maven Central Release -->
|
|
125 <!-- ================================================================== -->
|
|
126 <!-- -->
|
|
127 <!-- ================================================================== -->
|
|
128 <target name="release" depends="compile-core" description="--> generate the release distribution">
|
|
129 <copy file="${conf.dir}/nutch-default.xml"
|
|
130 todir="${build.classes}"/>
|
|
131 <copy file="${conf.dir}/nutch-site.xml"
|
|
132 todir="${build.classes}"/>
|
|
133
|
|
134 <!-- build the main artifact -->
|
|
135 <jar jarfile="${maven-jar}" basedir="${build.classes}" />
|
|
136
|
|
137 <!-- build the javadoc artifact -->
|
|
138 <javadoc
|
|
139 destdir="${release.dir}/javadoc"
|
|
140 overview="${src.dir}/overview.html"
|
|
141 author="true"
|
|
142 version="true"
|
|
143 use="true"
|
|
144 windowtitle="${name} ${version} API"
|
|
145 doctitle="${name} ${version} API"
|
|
146 bottom="Copyright &copy; ${year} The Apache Software Foundation"
|
|
147 >
|
|
148 <arg value="${javadoc.proxy.host}"/>
|
|
149 <arg value="${javadoc.proxy.port}"/>
|
|
150
|
|
151 <packageset dir="${src.dir}"/>
|
|
152 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
|
|
153 <packageset dir="${plugins.dir}/feed/src/java"/>
|
|
154 <packageset dir="${plugins.dir}/headings/src/java"/>
|
|
155 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
|
|
156 <packageset dir="${plugins.dir}/index-basic/src/java"/>
|
|
157 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
|
|
158 <packageset dir="${plugins.dir}/index-more/src/java"/>
|
|
159 <packageset dir="${plugins.dir}/index-static/src/java"/>
|
|
160 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
|
|
161 <packageset dir="${plugins.dir}/lib-http/src/java"/>
|
|
162 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
|
|
163 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
|
|
164 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
|
|
165 <packageset dir="${plugins.dir}/parse-html/src/java"/>
|
|
166 <packageset dir="${plugins.dir}/parse-js/src/java"/>
|
|
167 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
|
|
168 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
|
|
169 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
|
|
170 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
|
|
171 <packageset dir="${plugins.dir}/lib-http/src/java"/>
|
|
172 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
|
|
173 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
|
|
174 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
|
|
175 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
|
|
176 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
|
|
177 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
|
|
178 <packageset dir="${plugins.dir}/subcollection/src/java"/>
|
|
179 <packageset dir="${plugins.dir}/tld/src/java"/>
|
|
180 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
|
|
181 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
|
|
182 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
|
|
183 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
|
|
184 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
|
|
185 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
|
|
186 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
|
|
187 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
|
|
188 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
|
|
189 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
|
|
190 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
|
|
191 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
|
|
192
|
|
193 <link href="${javadoc.link.java}"/>
|
|
194 <link href="${javadoc.link.lucene}"/>
|
|
195 <link href="${javadoc.link.hadoop}"/>
|
|
196
|
|
197 <classpath refid="classpath"/>
|
|
198 <classpath>
|
|
199 <fileset dir="${plugins.dir}" >
|
|
200 <include name="**/*.jar"/>
|
|
201 </fileset>
|
|
202 </classpath>
|
|
203
|
|
204 <group title="Core" packages="org.apache.nutch.*"/>
|
|
205 <group title="Plugins API" packages="${plugins.api}"/>
|
|
206 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
|
|
207 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
|
|
208 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
|
|
209 <group title="Parse Plugins" packages="${plugins.parse}"/>
|
|
210 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
|
|
211 <group title="Misc. Plugins" packages="${plugins.misc}"/>
|
|
212 </javadoc>
|
|
213 <jar jarfile="${maven-javadoc-jar}">
|
|
214 <fileset dir="${release.dir}/javadoc" />
|
|
215 </jar>
|
|
216
|
|
217 <!-- build the sources artifact -->
|
|
218 <jar jarfile="${maven-sources-jar}">
|
|
219 <fileset dir="${src.dir}" />
|
|
220 </jar>
|
|
221 </target>
|
|
222
|
|
223 <!-- ================================================================== -->
|
|
224 <!-- Deploy to Apache Nexus -->
|
|
225 <!-- ================================================================== -->
|
|
226 <!-- -->
|
|
227 <!-- ================================================================== -->
|
|
228 <target name="deploy" depends="release" description="--> deploy to Apache Nexus">
|
|
229
|
|
230 <!-- generate a pom file -->
|
|
231 <ivy:makepom ivyfile="${ivy.file}" pomfile="${basedir}/pom.xml" templatefile="ivy/mvn.template">
|
|
232 <mapping conf="default" scope="compile"/>
|
|
233 <mapping conf="runtime" scope="runtime"/>
|
|
234 </ivy:makepom>
|
|
235
|
|
236 <!-- sign and deploy the main artifact -->
|
|
237 <artifact:mvn>
|
|
238 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
|
|
239 <arg value="-Durl=${maven-repository-url}" />
|
|
240 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
241 <arg value="-DpomFile=pom.xml" />
|
|
242 <arg value="-Dfile=${maven-jar}" />
|
|
243 <arg value="-Papache-release" />
|
|
244 </artifact:mvn>
|
|
245
|
|
246 <!-- sign and deploy the sources artifact -->
|
|
247 <artifact:mvn>
|
|
248 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
|
|
249 <arg value="-Durl=${maven-repository-url}" />
|
|
250 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
251 <arg value="-DpomFile=pom.xml" />
|
|
252 <arg value="-Dfile=${maven-sources-jar}" />
|
|
253 <arg value="-Dclassifier=sources" />
|
|
254 <arg value="-Papache-release" />
|
|
255 </artifact:mvn>
|
|
256
|
|
257 <!-- sign and deploy the javadoc artifact -->
|
|
258 <artifact:mvn>
|
|
259 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
|
|
260 <arg value="-Durl=${maven-repository-url}" />
|
|
261 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
262 <arg value="-DpomFile=pom.xml" />
|
|
263 <arg value="-Dfile=${maven-javadoc-jar}" />
|
|
264 <arg value="-Dclassifier=javadoc" />
|
|
265 <arg value="-Papache-release" />
|
|
266 </artifact:mvn>
|
|
267 </target>
|
|
268
|
|
269 <!-- ================================================================== -->
|
|
270 <!-- Make job jar -->
|
|
271 <!-- ================================================================== -->
|
|
272 <!-- -->
|
|
273 <!-- ================================================================== -->
|
|
274 <target name="job" depends="compile" description="--> make nutch.job jar">
|
|
275 <jar jarfile="${build.dir}/${final.name}.job">
|
|
276 <!-- If the build.classes has the nutch config files because the jar
|
|
277 command command has run, exclude them. The conf directory has
|
|
278 them.
|
|
279 -->
|
|
280 <zipfileset dir="${build.classes}"
|
|
281 excludes="nutch-default.xml,nutch-site.xml"/>
|
|
282 <zipfileset dir="${conf.dir}" excludes="*.template,hadoop*.*"/>
|
|
283 <zipfileset dir="${build.lib.dir}" prefix="lib"
|
|
284 includes="**/*.jar" excludes="hadoop-*.jar"/>
|
|
285 <zipfileset dir="${build.plugins}" prefix="classes/plugins"/>
|
|
286 </jar>
|
|
287 </target>
|
|
288
|
|
289 <target name="runtime" depends="jar, job" description="--> default target for running Nutch">
|
|
290 <mkdir dir="${runtime.dir}"/>
|
|
291 <mkdir dir="${runtime.local}"/>
|
|
292 <mkdir dir="${runtime.deploy}"/>
|
|
293 <!-- deploy area -->
|
|
294 <copy file="${build.dir}/${final.name}.job"
|
|
295 todir="${runtime.deploy}"/>
|
|
296 <copy todir="${runtime.deploy}/bin">
|
|
297 <fileset dir="src/bin"/>
|
|
298 </copy>
|
|
299 <chmod perm="ugo+x" type="file">
|
|
300 <fileset dir="${runtime.deploy}/bin"/>
|
|
301 </chmod>
|
|
302 <!-- local area -->
|
|
303 <copy file="${build.dir}/${final.name}.jar"
|
|
304 todir="${runtime.local}/lib"/>
|
|
305 <copy todir="${runtime.local}/lib/native">
|
|
306 <fileset dir="lib/native"/>
|
|
307 </copy>
|
|
308 <copy todir="${runtime.local}/conf">
|
|
309 <fileset dir="conf" excludes="*.template"/>
|
|
310 </copy>
|
|
311 <copy todir="${runtime.local}/bin">
|
|
312 <fileset dir="src/bin"/>
|
|
313 </copy>
|
|
314 <chmod perm="ugo+x" type="file">
|
|
315 <fileset dir="${runtime.local}/bin"/>
|
|
316 </chmod>
|
|
317 <copy todir="${runtime.local}/lib">
|
|
318 <fileset dir="${build.dir}/lib"/>
|
|
319 </copy>
|
|
320 <copy todir="${runtime.local}/plugins">
|
|
321 <fileset dir="${build.dir}/plugins"/>
|
|
322 </copy>
|
|
323 <copy todir="${runtime.local}/test">
|
|
324 <fileset dir="${build.dir}/test"/>
|
|
325 </copy>
|
|
326 </target>
|
|
327
|
|
328 <!-- ================================================================== -->
|
|
329 <!-- Compile test code -->
|
|
330 <!-- ================================================================== -->
|
|
331 <target name="compile-core-test" depends="compile-core, resolve-test" description="--> compile test code">
|
|
332 <javac
|
|
333 encoding="${build.encoding}"
|
|
334 srcdir="${test.src.dir}"
|
|
335 includes="org/apache/nutch/**/*.java"
|
|
336 destdir="${test.build.classes}"
|
|
337 debug="${javac.debug}"
|
|
338 optimize="${javac.optimize}"
|
|
339 target="${javac.version}"
|
|
340 source="${javac.version}"
|
|
341 deprecation="${javac.deprecation}">
|
|
342 <compilerarg value="-Xlint"/>
|
|
343 <classpath refid="test.classpath"/>
|
|
344 </javac>
|
|
345 </target>
|
|
346
|
|
347 <!-- ================================================================== -->
|
|
348 <!-- Run Nutch proxy -->
|
|
349 <!-- ================================================================== -->
|
|
350
|
|
351 <target name="proxy" depends="job, compile-core-test" description="--> run nutch proxy">
|
|
352 <java classname="org.apache.nutch.tools.proxy.TestbedProxy" fork="true">
|
|
353 <classpath refid="test.classpath"/>
|
|
354 <arg value="-fake"/>
|
|
355 <!--
|
|
356 <arg value="-delay"/>
|
|
357 <arg value="-200"/>
|
|
358 -->
|
|
359 <jvmarg line="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
360 </java>
|
|
361 </target>
|
|
362
|
|
363 <!-- ================================================================== -->
|
|
364 <!-- Run Nutch benchmarking analysis -->
|
|
365 <!-- ================================================================== -->
|
|
366
|
|
367 <target name="benchmark" description="--> run nutch benchmarking analysis">
|
|
368 <java classname="org.apache.nutch.tools.Benchmark" fork="true">
|
|
369 <classpath refid="test.classpath"/>
|
|
370 <jvmarg line="-Xmx512m -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
371 <arg value="-maxPerHost"/>
|
|
372 <arg value="10"/>
|
|
373 <arg value="-seeds"/>
|
|
374 <arg value="1"/>
|
|
375 <arg value="-depth"/>
|
|
376 <arg value="5"/>
|
|
377 </java>
|
|
378 </target>
|
|
379
|
|
380 <!-- ================================================================== -->
|
|
381 <!-- Run unit tests -->
|
|
382 <!-- ================================================================== -->
|
|
383 <target name="test" depends="test-core, test-plugins" description="--> run JUnit tests"/>
|
|
384
|
|
385 <target name="test-core" depends="job, compile-core-test" description="--> run core JUnit tests only">
|
|
386
|
|
387 <delete dir="${test.build.data}"/>
|
|
388 <mkdir dir="${test.build.data}"/>
|
|
389 <!--
|
|
390 copy resources needed in junit tests
|
|
391 -->
|
|
392 <copy todir="${test.build.data}">
|
|
393 <fileset dir="src/testresources" includes="**/*"/>
|
|
394 </copy>
|
|
395
|
|
396 <copy file="${test.src.dir}/nutch-site.xml"
|
|
397 todir="${test.build.classes}"/>
|
|
398
|
|
399 <copy file="${test.src.dir}/log4j.properties"
|
|
400 todir="${test.build.classes}"/>
|
|
401
|
|
402 <copy file="${test.src.dir}/crawl-tests.xml"
|
|
403 todir="${test.build.classes}"/>
|
|
404
|
|
405 <copy file="${test.src.dir}/domain-urlfilter.txt"
|
|
406 todir="${test.build.classes}"/>
|
|
407
|
|
408 <copy file="${test.src.dir}/filter-all.txt"
|
|
409 todir="${test.build.classes}"/>
|
|
410
|
|
411 <junit printsummary="yes" haltonfailure="no" fork="yes" dir="${basedir}"
|
|
412 errorProperty="tests.failed" failureProperty="tests.failed" maxmemory="1000m">
|
|
413 <sysproperty key="test.build.data" value="${test.build.data}"/>
|
|
414 <sysproperty key="test.src.dir" value="${test.src.dir}"/>
|
|
415 <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
416 <classpath refid="test.classpath"/>
|
|
417 <formatter type="${test.junit.output.format}" />
|
|
418 <batchtest todir="${test.build.dir}" unless="testcase">
|
|
419 <fileset dir="${test.src.dir}"
|
|
420 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
|
|
421 </batchtest>
|
|
422 <batchtest todir="${test.build.dir}" if="testcase">
|
|
423 <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
|
|
424 </batchtest>
|
|
425 </junit>
|
|
426
|
|
427 <fail if="tests.failed">Tests failed!</fail>
|
|
428
|
|
429 </target>
|
|
430
|
|
431 <target name="test-plugins" depends="compile" description="--> run plugin JUnit tests only">
|
|
432 <ant dir="src/plugin" target="test" inheritAll="false"/>
|
|
433 </target>
|
|
434
|
|
435 <target name="nightly" depends="test, tar-src, zip-src" description="--> run the nightly target build">
|
|
436 </target>
|
|
437
|
|
438 <!-- ================================================================== -->
|
|
439 <!-- Ivy targets -->
|
|
440 <!-- ================================================================== -->
|
|
441
|
|
442 <!-- target: resolve ================================================= -->
|
|
443 <target name="resolve-default" depends="clean-lib" description="--> resolve and retrieve dependencies with ivy">
|
|
444 <ivy:resolve file="${ivy.file}" conf="default" log="download-only"/>
|
|
445 <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
|
|
446 <antcall target="copy-libs"/>
|
|
447 </target>
|
|
448
|
|
449 <target name="resolve-test" depends="clean-lib" description="--> resolve and retrieve dependencies with ivy">
|
|
450 <ivy:resolve file="${ivy.file}" conf="test" log="download-only"/>
|
|
451 <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
|
|
452 <antcall target="copy-libs"/>
|
|
453 </target>
|
|
454
|
|
455 <target name="copy-libs" description="--> copy the libs in lib, which are not ivy enabled">
|
|
456 <!-- copy the libs in lib, which are not ivy enabled-->
|
|
457 <copy todir="${build.lib.dir}/" failonerror="false">
|
|
458 <fileset dir="${lib.dir}" includes="**/*.jar"/>
|
|
459 </copy>
|
|
460 </target>
|
|
461
|
|
462 <!-- target: publish-local =========================================== -->
|
|
463 <target name="publish-local" depends="jar" description="--> publish this project in the local ivy repository">
|
|
464 <ivy:publish artifactspattern="${build.dir}/[artifact]-${version}.[ext]"
|
|
465 resolver="local"
|
|
466 pubrevision="${version}"
|
|
467 pubdate="${now}"
|
|
468 status="integration"
|
|
469 forcedeliver="true"
|
|
470 overwrite="true"
|
|
471 />
|
|
472 <echo message="project ${ant.project.name} published locally with version ${version}" />
|
|
473 </target>
|
|
474
|
|
475 <!-- target: report ================================================== -->
|
|
476 <target name="report" depends="resolve-test" description="--> generates a report of dependencies">
|
|
477 <ivy:report todir="${build.dir}"/>
|
|
478 </target>
|
|
479
|
|
480 <!-- target: ivy-init ================================================ -->
|
|
481 <target name="ivy-init" depends="ivy-probe-antlib, ivy-init-antlib" description="--> initialise Ivy settings">
|
|
482 <ivy:settings file="${ivy.dir}/ivysettings.xml" />
|
|
483 </target>
|
|
484
|
|
485 <!-- target: ivy-probe-antlib ======================================== -->
|
|
486 <target name="ivy-probe-antlib" description="--> probe the antlib library">
|
|
487 <condition property="ivy.found">
|
|
488 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
|
|
489 </condition>
|
|
490 </target>
|
|
491
|
|
492 <!-- target: ivy-download ============================================ -->
|
|
493 <target name="ivy-download" description="--> download ivy">
|
|
494 <available file="${ivy.jar}" property="ivy.jar.found"/>
|
|
495 <antcall target="ivy-download-unchecked"/>
|
|
496 </target>
|
|
497
|
|
498 <!-- target: ivy-download-unchecked ================================== -->
|
|
499 <target name="ivy-download-unchecked" unless="ivy.jar.found" description="--> fetch any ivy file">
|
|
500 <get src="${ivy.repo.url}" dest="${ivy.jar}" usetimestamp="true" />
|
|
501 </target>
|
|
502
|
|
503 <!-- target: ivy-init-antlib ========================================= -->
|
|
504 <target name="ivy-init-antlib" depends="ivy-download" unless="ivy.found" description="--> attempt to use Ivy with Antlib">
|
|
505 <typedef uri="antlib:org.apache.ivy.ant" onerror="fail" loaderRef="ivyLoader">
|
|
506 <classpath>
|
|
507 <pathelement location="${ivy.jar}" />
|
|
508 </classpath>
|
|
509 </typedef>
|
|
510 <fail>
|
|
511 <condition>
|
|
512 <not>
|
|
513 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
|
|
514 </not>
|
|
515 </condition>
|
|
516 You need Apache Ivy 2.0 or later from http://ant.apache.org/
|
|
517 It could not be loaded from ${ivy.repo.url}
|
|
518 </fail>
|
|
519 </target>
|
|
520
|
|
521 <!-- ================================================================== -->
|
|
522 <!-- Documentation -->
|
|
523 <!-- ================================================================== -->
|
|
524 <target name="javadoc" depends="compile" description="--> generate Javadoc">
|
|
525 <mkdir dir="${build.javadoc}"/>
|
|
526 <javadoc
|
|
527 overview="${src.dir}/overview.html"
|
|
528 destdir="${build.javadoc}"
|
|
529 author="true"
|
|
530 version="true"
|
|
531 use="true"
|
|
532 windowtitle="${name} ${version} API"
|
|
533 doctitle="${name} ${version} API"
|
|
534 bottom="Copyright &copy; ${year} The Apache Software Foundation"
|
|
535 >
|
|
536 <arg value="${javadoc.proxy.host}"/>
|
|
537 <arg value="${javadoc.proxy.port}"/>
|
|
538
|
|
539 <packageset dir="${src.dir}"/>
|
|
540 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
|
|
541 <packageset dir="${plugins.dir}/feed/src/java"/>
|
|
542 <packageset dir="${plugins.dir}/headings/src/java"/>
|
|
543 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
|
|
544 <packageset dir="${plugins.dir}/index-basic/src/java"/>
|
|
545 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
|
|
546 <packageset dir="${plugins.dir}/index-more/src/java"/>
|
|
547 <packageset dir="${plugins.dir}/index-static/src/java"/>
|
|
548 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
|
|
549 <packageset dir="${plugins.dir}/lib-http/src/java"/>
|
|
550 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
|
|
551 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
|
|
552 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
|
|
553 <packageset dir="${plugins.dir}/parse-html/src/java"/>
|
|
554 <packageset dir="${plugins.dir}/parse-js/src/java"/>
|
|
555 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
|
|
556 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
|
|
557 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
|
|
558 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
|
|
559 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
|
|
560 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
|
|
561 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
|
|
562 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
|
|
563 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
|
|
564 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
|
|
565 <packageset dir="${plugins.dir}/subcollection/src/java"/>
|
|
566 <packageset dir="${plugins.dir}/tld/src/java"/>
|
|
567 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
|
|
568 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
|
|
569 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
|
|
570 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
|
|
571 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
|
|
572 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
|
|
573 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
|
|
574 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
|
|
575 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
|
|
576 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
|
|
577 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
|
|
578
|
|
579 <link href="${javadoc.link.java}"/>
|
|
580 <link href="${javadoc.link.lucene}"/>
|
|
581 <link href="${javadoc.link.hadoop}"/>
|
|
582
|
|
583 <classpath refid="classpath"/>
|
|
584 <classpath>
|
|
585 <fileset dir="${plugins.dir}" >
|
|
586 <include name="**/*.jar"/>
|
|
587 </fileset>
|
|
588 </classpath>
|
|
589
|
|
590 <group title="Core" packages="org.apache.nutch.*"/>
|
|
591 <group title="Plugins API" packages="${plugins.api}"/>
|
|
592 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
|
|
593 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
|
|
594 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
|
|
595 <group title="Parse Plugins" packages="${plugins.parse}"/>
|
|
596 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
|
|
597 <group title="Misc. Plugins" packages="${plugins.misc}"/>
|
|
598 </javadoc>
|
|
599 <!-- Copy the plugin.dtd file to the plugin doc-files dir -->
|
|
600 <copy file="${plugins.dir}/plugin.dtd"
|
|
601 todir="${build.javadoc}/org/apache/nutch/plugin/doc-files"/>
|
|
602 </target>
|
|
603
|
|
604 <target name="default-doc" description="--> generate default Nutch documentation">
|
|
605 <style basedir="${conf.dir}" destdir="${docs.dir}"
|
|
606 includes="nutch-default.xml" style="conf/nutch-conf.xsl"/>
|
|
607 </target>
|
|
608
|
|
609 <!-- ================================================================== -->
|
|
610 <!-- D I S T R I B U T I O N -->
|
|
611 <!-- ================================================================== -->
|
|
612 <!-- -->
|
|
613 <!-- ================================================================== -->
|
|
614 <target name="package-src" depends="runtime, javadoc" description="--> generate source distribution package">
|
|
615 <mkdir dir="${dist.dir}"/>
|
|
616 <mkdir dir="${src.dist.version.dir}"/>
|
|
617 <mkdir dir="${src.dist.version.dir}/lib"/>
|
|
618 <mkdir dir="${src.dist.version.dir}/docs"/>
|
|
619 <mkdir dir="${src.dist.version.dir}/docs/api"/>
|
|
620 <mkdir dir="${src.dist.version.dir}/ivy"/>
|
|
621
|
|
622 <copy todir="${src.dist.version.dir}/lib" includeEmptyDirs="false">
|
|
623 <fileset dir="lib"/>
|
|
624 </copy>
|
|
625
|
|
626 <copy todir="${src.dist.version.dir}/conf">
|
|
627 <fileset dir="${conf.dir}" excludes="**/*.template"/>
|
|
628 </copy>
|
|
629
|
|
630 <copy todir="${src.dist.version.dir}/docs/api">
|
|
631 <fileset dir="${build.javadoc}"/>
|
|
632 </copy>
|
|
633
|
|
634 <copy todir="${src.dist.version.dir}">
|
|
635 <fileset dir=".">
|
|
636 <include name="*.txt" />
|
|
637 <!--<include name="KEYS" />-->
|
|
638 </fileset>
|
|
639 </copy>
|
|
640
|
|
641 <copy todir="${src.dist.version.dir}/src" includeEmptyDirs="true">
|
|
642 <fileset dir="src"/>
|
|
643 </copy>
|
|
644
|
|
645 <copy todir="${src.dist.version.dir}/ivy" includeEmptyDirs="true">
|
|
646 <fileset dir="ivy"/>
|
|
647 </copy>
|
|
648
|
|
649 <copy todir="${src.dist.version.dir}/" file="build.xml"/>
|
|
650 <copy todir="${src.dist.version.dir}/" file="default.properties"/>
|
|
651
|
|
652 </target>
|
|
653
|
|
654 <target name="package-bin" depends="runtime, javadoc" description="--> generate binary distribution package">
|
|
655 <mkdir dir="${dist.dir}"/>
|
|
656 <mkdir dir="${bin.dist.version.dir}"/>
|
|
657 <mkdir dir="${bin.dist.version.dir}/lib"/>
|
|
658 <mkdir dir="${bin.dist.version.dir}/bin"/>
|
|
659 <mkdir dir="${bin.dist.version.dir}/conf"/>
|
|
660 <mkdir dir="${bin.dist.version.dir}/docs"/>
|
|
661 <mkdir dir="${bin.dist.version.dir}/docs/api"/>
|
|
662 <mkdir dir="${bin.dist.version.dir}/plugins"/>
|
|
663
|
|
664 <copy todir="${bin.dist.version.dir}/lib" includeEmptyDirs="false">
|
|
665 <fileset dir="runtime/local/lib"/>
|
|
666 </copy>
|
|
667
|
|
668 <copy todir="${bin.dist.version.dir}/bin">
|
|
669 <fileset dir="runtime/local/bin"/>
|
|
670 </copy>
|
|
671
|
|
672 <chmod perm="ugo+x" type="file">
|
|
673 <fileset dir="${bin.dist.version.dir}/bin"/>
|
|
674 </chmod>
|
|
675
|
|
676 <copy todir="${bin.dist.version.dir}/conf">
|
|
677 <fileset dir="runtime/local/conf" excludes="**/*.template"/>
|
|
678 </copy>
|
|
679
|
|
680 <copy todir="${bin.dist.version.dir}/docs/api">
|
|
681 <fileset dir="${build.javadoc}"/>
|
|
682 </copy>
|
|
683
|
|
684 <copy todir="${bin.dist.version.dir}">
|
|
685 <fileset dir=".">
|
|
686 <include name="*.txt" />
|
|
687 </fileset>
|
|
688 </copy>
|
|
689
|
|
690 <copy todir="${bin.dist.version.dir}/plugins" includeEmptyDirs="true">
|
|
691 <fileset dir="runtime/local/plugins"/>
|
|
692 </copy>
|
|
693
|
|
694 </target>
|
|
695
|
|
696 <!-- ================================================================== -->
|
|
697 <!-- Make src release tarball -->
|
|
698 <!-- ================================================================== -->
|
|
699 <target name="tar-src" depends="package-src" description="--> generate src.tar.gz distribution package">
|
|
700 <tar compression="gzip" longfile="gnu"
|
|
701 destfile="${src.dist.version.dir}.tar.gz">
|
|
702 <tarfileset dir="${src.dist.version.dir}" mode="664" prefix="${final.name}">
|
|
703 <exclude name="src/bin/*" />
|
|
704 <include name="**" />
|
|
705 </tarfileset>
|
|
706 <tarfileset dir="${src.dist.version.dir}" mode="755" prefix="${final.name}">
|
|
707 <include name="src/bin/*" />
|
|
708 </tarfileset>
|
|
709 </tar>
|
|
710 </target>
|
|
711
|
|
712 <!-- ================================================================== -->
|
|
713 <!-- Make bin release tarball -->
|
|
714 <!-- ================================================================== -->
|
|
715 <target name="tar-bin" depends="package-bin" description="--> generate bin.tar.gz distribution package">
|
|
716 <tar compression="gzip" longfile="gnu"
|
|
717 destfile="${bin.dist.version.dir}.tar.gz">
|
|
718 <tarfileset dir="${bin.dist.version.dir}" mode="664" prefix="${final.name}">
|
|
719 <exclude name="bin/*" />
|
|
720 <include name="**" />
|
|
721 </tarfileset>
|
|
722 <tarfileset dir="${bin.dist.version.dir}" mode="755" prefix="${final.name}">
|
|
723 <include name="bin/*" />
|
|
724 </tarfileset>
|
|
725 </tar>
|
|
726 </target>
|
|
727
|
|
728 <!-- ================================================================== -->
|
|
729 <!-- Make src release zip -->
|
|
730 <!-- ================================================================== -->
|
|
731 <target name="zip-src" depends="package-src" description="--> generate src.zip distribution package">
|
|
732 <zip compress="true" casesensitive="yes"
|
|
733 destfile="${src.dist.version.dir}.zip">
|
|
734 <zipfileset dir="${src.dist.version.dir}" filemode="664" prefix="${final.name}">
|
|
735 <exclude name="src/bin/*" />
|
|
736 <include name="**" />
|
|
737 </zipfileset>
|
|
738 <zipfileset dir="${src.dist.version.dir}" filemode="755" prefix="${final.name}">
|
|
739 <include name="src/bin/*" />
|
|
740 </zipfileset>
|
|
741 </zip>
|
|
742 </target>
|
|
743
|
|
744 <!-- ================================================================== -->
|
|
745 <!-- Make bin release zip -->
|
|
746 <!-- ================================================================== -->
|
|
747 <target name="zip-bin" depends="package-bin" description="--> generate bin.zip distribution package">
|
|
748 <zip compress="true" casesensitive="yes"
|
|
749 destfile="${bin.dist.version.dir}.zip">
|
|
750 <zipfileset dir="${bin.dist.version.dir}" filemode="664" prefix="${final.name}">
|
|
751 <exclude name="bin/*" />
|
|
752 <include name="**" />
|
|
753 </zipfileset>
|
|
754 <zipfileset dir="${bin.dist.version.dir}" filemode="755" prefix="${final.name}">
|
|
755 <include name="bin/*" />
|
|
756 </zipfileset>
|
|
757 </zip>
|
|
758 </target>
|
|
759
|
|
760 <!-- ================================================================== -->
|
|
761 <!-- Clean. Delete the build files, and their directories -->
|
|
762 <!-- ================================================================== -->
|
|
763
|
|
764 <!-- target: clean =================================================== -->
|
|
765 <target name="clean" depends="clean-build, clean-lib, clean-dist, clean-runtime" description="--> clean the project" />
|
|
766
|
|
767 <!-- target: clean-local ============================================= -->
|
|
768 <target name="clean-local" depends=""
|
|
769 description="--> cleans the local repository for the current module">
|
|
770 <delete dir="${ivy.local.default.root}/${ivy.organisation}/${ivy.module}"/>
|
|
771 </target>
|
|
772
|
|
773 <!-- target: clean-lib =============================================== -->
|
|
774 <target name="clean-lib" description="--> clean the project libraries directory (dependencies)">
|
|
775 <delete includeemptydirs="true" dir="${build.lib.dir}"/>
|
|
776 </target>
|
|
777
|
|
778 <!-- target: clean-build ============================================= -->
|
|
779 <target name="clean-build" description="--> clean the project built files">
|
|
780 <delete includeemptydirs="true" dir="${build.dir}"/>
|
|
781 </target>
|
|
782
|
|
783 <!-- target: clean-dist ============================================= -->
|
|
784 <target name="clean-dist" description="--> clean the project dist files">
|
|
785 <delete includeemptydirs="true" dir="${dist.dir}"/>
|
|
786 </target>
|
|
787
|
|
788 <!-- target: clean-cache ============================================= -->
|
|
789 <target name="clean-cache" depends=""
|
|
790 description="--> delete ivy cache">
|
|
791 <ivy:cleancache />
|
|
792 </target>
|
|
793
|
|
794 <target name="clean-runtime" description="--> clean the project runtime area">
|
|
795 <delete includeemptydirs="true" dir="${runtime.dir}"/>
|
|
796 </target>
|
|
797
|
|
798 <!-- ================================================================== -->
|
|
799 <!-- RAT targets -->
|
|
800 <!-- ================================================================== -->
|
|
801 <target name="rat-sources-typedef" description="--> run RAT antlib task">
|
|
802 <typedef resource="org/apache/rat/anttasks/antlib.xml" >
|
|
803 <classpath>
|
|
804 <fileset dir="." includes="rat*.jar"/>
|
|
805 </classpath>
|
|
806 </typedef>
|
|
807 </target>
|
|
808
|
|
809 <target name="rat-sources" depends="rat-sources-typedef"
|
|
810 description="--> runs RAT tasks over src/java">
|
|
811 <rat:report xmlns:rat="antlib:org.apache.rat.anttasks">
|
|
812 <fileset dir="src">
|
|
813 <include name="java/**/*"/>
|
|
814 <include name="plugin/**/src/**/*"/>
|
|
815 </fileset>
|
|
816 </rat:report>
|
|
817 </target>
|
|
818
|
|
819 <!-- ================================================================== -->
|
|
820 <!-- SONAR targets -->
|
|
821 <!-- ================================================================== -->
|
|
822
|
|
823 <!-- Define the Sonar task if this hasn't been done in a common script -->
|
|
824 <taskdef uri="antlib:org.sonar.ant" resource="org/sonar/ant/antlib.xml">
|
|
825 <classpath path="${ant.library.dir}"/>
|
|
826 <classpath path="${mysql.library.dir}"/>
|
|
827 </taskdef>
|
|
828
|
|
829 <!-- Add the target -->
|
|
830 <target name="sonar" description="--> run SONAR analysis">
|
|
831
|
|
832 <!-- list of mandatory source directories (required) -->
|
|
833 <property name="sonar.sources" value="${src.dir}"/>
|
|
834
|
|
835 <!-- list of properties (optional) -->
|
|
836 <property name="sonar.projectName" value="Nutch Trunk 1.4 Sonar Analysis" />
|
|
837 <property name="sonar.binaries" value="${build.dir}/classes" />
|
|
838 <property name="sonar.binaries" value="${build.dir}/plugins" />
|
|
839 <property name="sonar.tests" value="${test.src.dir}" />
|
|
840
|
|
841 <sonar:sonar workDir="${base.dir}" key="org.apache.nutch:trunk"
|
|
842 version="1.4-SNAPSHOT" xmlns:sonar="antlib:org.sonar.ant"/>
|
|
843 </target>
|
|
844
|
|
845 </project>
|