package org.varienaja.util.wikipedia;
   
   import java.io.IOException;
   import java.io.InputStream;
   import java.net.MalformedURLException;
   import java.net.URL;
   import java.net.URLConnection;
   import java.util.HashMap;
   import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import org.apache.log4j.Logger;
  import org.varienaja.util.LRUMap;
  import org.varienaja.util.StringUtil;
  import org.w3c.dom.Document;
  import org.w3c.dom.NamedNodeMap;
  import org.w3c.dom.Node;
  import org.w3c.dom.NodeList;
  import org.xml.sax.InputSource;
  import org.xml.sax.SAXException;
  
  import com.sun.org.apache.xerces.internal.parsers.DOMParser;
  
  /**
   * Class that provides functions to search in Wikipedia.
   * @author Varienaja
   */
  public class WikipediaSearcher {
  	private static final Logger log = Logger.getLogger(WikipediaSearcher.class);
  	
  	/**
  	 * Base URL for searching Wikipedia. This URL return all links as xml.
  	 */
  	private static final String BASEURL = "http://en.wikipedia.org/w/api.php?action=query&prop=links&format=xml&pllimit=500&titles=";
  	private static final String CONTENTURL = "http://en.wikipedia.org/w/api.php?action=query&prop=revisions&format=xml&rvprop=content&titles=";
  	private static final String NORMALURL = "http://en.wikipedia.org/wiki/";
  	
  	/**
  	 * Cache for bandname/playlist --> genres.
  	 */
  	private static Map<String,String[]> _keywordcache = new HashMap<String,String[]>(10);
  	/**
  	 * Cache for bandnames --> bandurl.
  	 */
  	private static Map<String,String> _bandnamecache = new LRUMap<String,String>("WikipediaBandEntries",10);
  
  	/**
  	 * <p>Searches Wikipedia for Genre(s) of a specific playlist. Results of
  	 * this method are cached, so you can call this method safely and
  	 * performantly for all songs of a playlist.</p>
  	 * @param bandname The name of the Band
  	 * @param playlistname The name of the Playlist
  	 * @return An array of Strings, describing the Genre(s) of the Playlist
  	 */
  	public static String[] getKeywords (String bandname, String playlistname) {
  		String[] cached = _keywordcache.get(bandname+" / "+playlistname);
  		if (cached==null) {
  			log.debug("Cache miss for: "+bandname+", playlist: "+playlistname+". Searching Wikipedia.org");
  			cached = getKeywordsInternal(bandname,playlistname);
  			_keywordcache.put(bandname+" / "+playlistname,cached);
  		} else {
  			log.debug("Cache hit for: "+bandname+", playlist: "+playlistname);
  		}
  		return cached;
  	}
  	
  	/**
  	 * Searches Wikipedia for the URL of the given band.
  	 * TODO Refactor repetitive code.
  	 * @param bandname The Band to search for.
  	 * @return The String that can be appended to the stardard-Wikipedia URL to
  	 * open the page for the Band. The result is null, of no band was found.
  	 * @throws WikipediaException When searching went wrong somehow.
  	 */
  	protected static String getBandURL(String bandname) throws WikipediaException {
  		String cached = _bandnamecache.get(bandname);
  		if (cached==null) {
  			log.debug("Cache miss for: "+bandname+". Searching Wikipedia.org");
  			Document doc = null;
  			List<String> links = null;
  			String candidate = bandname;
  			
  			//Step 1
  			doc = queryWikipedia(getSearchURL(candidate));
  			if (doc==null) {
  				throw new WikipediaException("Wikipedia unavailable? Check your internet connection.");
  			}
  			
  			//Step 2
  			links = getLinksFromDocument(doc);
  			
  			//If we only get or an explicit Wikipedia:Redirect result, we're probably being redirected: follow the first link.
  			if (links.size()==1 || links.contains("Wikipedia:Redirect")) {
  				candidate = links.get(0);
  				doc = queryWikipedia(getSearchURL(candidate));
  				links = getLinksFromDocument(doc);
 			}
 			
 			String origCandidate = candidate;
 			candidate = bandname+" (band)";
 			if (StringUtil.contains(links,candidate)) { //Try (band)-suffix
 				_bandnamecache.put(bandname, candidate);
 				return candidate;
 			}
 				candidate = bandname+" (entertainer)";
 				if (StringUtil.contains(links,candidate)) {
 					_bandnamecache.put(bandname, candidate);
 					return candidate;
 				} else {
 					candidate = bandname+" (disambiguation)";
 					if (StringUtil.contains(links,candidate)) {
 						doc = queryWikipedia(getSearchURL(candidate));
 						links = getLinksFromDocument(doc);
 						candidate = bandname+" (band)";
 						if (StringUtil.contains(links,candidate)) {
 							_bandnamecache.put(bandname, candidate);
 							return candidate;
 						}
 					}
 				}
 			_bandnamecache.put(bandname, origCandidate);
 			return origCandidate;
 		} else {
 			log.debug("Cache hit for: "+bandname+".");
 			return cached;
 		}
 	}
 	
 	/**
 	 * <p>Searches Wikipedia for Genre(s) of a specific playlist. The strategy
 	 * is as follows:</p>
 	 * <ol>
 	 *    <li>We search for "bandname"</li>
 	 *    <li>If the resulting Document has no links, we try "bandname (band)".
 	 *        If the resulting Document has a link "bandname (disambiguation)"
 	 *        or "bandname (entertainer)" we will follow that one and do this
 	 *        step again.<br/>
 	 *        If the resulting Document has a link "bandname (band)", we will
 	 *        follow that link.</br>
 	 *        Otherwise, wo do nothing. (We're probably right where we want to
 	 *        be.)</li>
 	 *    <li>We scan for a link to "bandname discograpy", and follow it if
 	 *        present. Then, we scan for a link to the playlist or to 
 	 *        "playlist (album) on this page. If it is present we follow this
 	 *        link. If there isn't a link to a Playlist, we return null.
 	 *        </li>
 	 *    <li>We are on the page of the Playlist. We search for the genres and
 	 *        return them.</li> 
 	 * </ol> 
 	 * @param bandname The name of the Band
 	 * @param playlistname The name of the Playlist
 	 * @return An array of Strings, describing the Genre(s) of the Playlist
 	 */
 	private static String[] getKeywordsInternal(String bandname, String playlistname) {
 		String[] result = new String[0];
 		Document doc = null;
 		List<String> results = null;
 		
 		//Step 1+2
 		try {
 			String url = getBandURL(bandname);
 			log.debug("Wikipedia URL for band: " + bandname+" is: "+url);
 			doc = queryWikipedia(getSearchURL(url));
 		} catch (WikipediaException e) {
 			log.error(e.toString());
 			return result; //Return immediately, if communication with Wikipedia failed
 		}
 		
 		//Step 3
 		results = getLinksFromDocument(doc);
 		//TODO Create a contains that returns the item closest to the query?
 		if (StringUtil.contains(results,bandname+" discography")) {
 			doc = queryWikipedia(getSearchURL(bandname+" discography"));
 			results = getLinksFromDocument(doc);
 		}
 		if (StringUtil.contains(results,playlistname+" (album)")) {
 			doc = queryWikipedia(getContentURL(playlistname+" (album)"));
 		} else {
 			if (StringUtil.contains(results,playlistname)) {
 				doc = queryWikipedia(getContentURL(playlistname));
 			} else {
 				return result;
 			}
 		}
 		
 		//Step 4
 		String content = getContentFromDocument(doc);
 		result = extractGenrePart(content);
 		return result;
 	}
 	
 	/**
 	 * Returns the Document object which was created from the xml in the
 	 * inputstream.
 	 * @param in The InputStream containing XML-data
 	 * @return The w3c.dom.Document
 	 */
 	protected static Document getDocumentFromInputStream(InputStream in) {
 		InputSource source = new InputSource(in);
 		DOMParser parser = new DOMParser();
 		try {
 			parser.parse(source);
 			Document doc = parser.getDocument();
 			return doc;
 		} catch (SAXException e) {
 			log.error("Error processing InputStream: "+e.getMessage());
 		} catch (IOException e) {
 			log.error("Error processing InputStream: "+e.getMessage());
 		}
 		return null;
 	}
 	
 	/**
 	 * Returns all links that are present in a certain InputStream. You should
 	 * only call this method with InputStream-parameters with xml-data in them
 	 * that comes from Wikipedia.org
 	 * @param in The InputStream containing XML-data
 	 * @return All (wikipedia)links that are found in the InputStream, or null
 	 *         if an error occured.
 	 */
 	protected static List<String> getLinks(InputStream in) {
 		Document doc = getDocumentFromInputStream(in);
 		return doc==null ? null : getLinksFromDocument(doc);
 	}
 	
 	/**
 	 * Returns all links that are present in a Document. You should
 	 * only call this method with Document-parameters with data in them
 	 * that comes from Wikipedia.org
 	 * TODO Use XPath?
 	 * @param doc The document
 	 * @return All (wikipedia)links that are found in the Document
 	 */
 	protected static List<String> getLinksFromDocument(Document doc) {
 		List<String> results = new LinkedList<String>();
 		NodeList list = doc.getElementsByTagName("links");
 		for (int i=0; i<list.getLength(); i++) {
 			Node node = list.item(i);
 			NodeList children = node.getChildNodes();
 			for (int j=0; j<children.getLength(); j++) {
 				Node child = children.item(j);
 				NamedNodeMap map = child.getAttributes();
 				Node title = map.getNamedItem("title");
 				results.add(title.getTextContent());
 			}
 		}
 		return results;
 	}
 	
 	/**
 	 * Returns the content-element of an XMLDocument.
 	 * @param doc The Document
 	 * @return Everything between &lt;content&gt; and &lt;/content&gt;
 	 */
 	protected static String getContentFromDocument(Document doc) {
 		StringBuilder sb = new StringBuilder();
 		NodeList list = doc.getElementsByTagName("rev");
 		for (int i=0; i<list.getLength(); i++) {
 			Node node = list.item(i);
 			sb.append(node.getTextContent());
 		}
 		return sb.toString();
 	}
 	
 	/**
 	 * Constructs a URL for searching Wikipedia
 	 * @param query The query
 	 * @return The URL
 	 */
 	protected static String getSearchURL(String query) {
 		return (BASEURL+query).replaceAll(" ","%20");
 	}
 	
 	/**
 	 * Constructs a URL for getting contents from Wikipedia
 	 * @param query The query
 	 * @return The URL
 	 */
 	protected static String getContentURL(String query) {
 		return (CONTENTURL+query).replaceAll(" ","%20");
 	}
 	
 	/**
 	 * Searches Wikipedia, and returns the resulting page as a
 	 * completely parsed XMLDocument
 	 * @param location The URL
 	 * @return The Document, containing the links of the page.
 	 */
 	protected static Document queryWikipedia(String location) {
 		try {
 			URL url = new URL(location);
 			URLConnection urlc = url.openConnection();
 			urlc.setRequestProperty("user-agent","Mozilla/5.0"); //TODO See if this is nessecary
 			InputStream in = urlc.getInputStream();
 			InputSource source = new InputSource(in);
 			DOMParser parser = new DOMParser();
 			parser.parse(source);
 			in.close();
 			return parser.getDocument();
 		} catch (MalformedURLException e) {
 			log.error("Error searching Wikipedia: "+e.getMessage());
 		} catch (IOException e) {
 			log.error("Error searching Wikipedia: "+e.getMessage());
 		} catch (SAXException e) {
 			log.error("Error searching Wikipedia: "+e.getMessage());
 		}
 		return null;
 	}
 	
 	/**
 	 * Searches for a substring like 
 	 * "Genre       =[[Rhythm and blues|R&amp;B]], [[Funk]], [[Rock music|Rock]]"
 	 * in the content.
 	 * @param content The String to search in
 	 * @return A String Array, containing the Keywords found.
 	 */
 	protected static String[] extractGenrePart(String content) {
 		Pattern pattern = Pattern.compile("Genre\\s*=\\s*(\\[\\[.*?\\]\\]).?\\s\\|");
 		Matcher matcher = pattern.matcher(content);
 		if (matcher.find()) {
 			if (log.isDebugEnabled()) {
 				for (int i=0;i<=matcher.groupCount();i++) {
 					log.debug(matcher.group(i));
 				}
 			}
 			if (matcher.groupCount()==1) {
 				return extractGenres(matcher.group(1));
 			}
 		}
 		return new String[0];
 	}
 	
 	/**
 	 * <p>Processes a String like "[[Rhythm and blues|R&amp;B]], [[Funk]], [[Rock music|Rock]]"
 	 * and creates a String Array, containing only the text.</p>
 	 * <p>[[X]] becomes X<br/>
 	 * [[X|Y]] becomes Y</p>
 	 * TODO Create unittests for this method, and use a different method for the splitting
 	 * @param genrespart The inputstring
 	 * @return An Array of String containing the Keywords found in the inputstring.
 	 */
 	protected static String[] extractGenres(String genrespart) {
 		if (genrespart==null) {
 			return null;
 		}
 		
 		List<String> result = new LinkedList<String>();
 		String[] items = genrespart.split(",\\s*|<br>|/");
 		for (int i=0;i<items.length;i++) {
 			int start = items[i].indexOf("|");
 			if (start==-1) {
 				start = items[i].lastIndexOf("[");
 			}
 			int end = items[i].indexOf("]");
 			if (end!=-1) {
 				result.add(items[i].substring(start+1,end).replaceAll("&amp;","&"));
 			}
 		}
 		
 		return result.toArray(new String[result.size()]);
 	}
 	
 	/**
 	 * Retrieves Band info from Wikipedia. Bandinfo is just the general contents
 	 * of the Wikipedia-entry.
 	 * @param bandname
 	 * @return The first paragraph of the Wikipedia entry for the given Band.
 	 * @throws WikipediaException When no info was found or the contents returned
 	 * could not be deciphered.
 	 */
 	public static String getBandInfo(String bandname) throws WikipediaException {
 		String url;
 		try {
 			url = getContentURL(getBandURL(bandname));
 			Document doc = queryWikipedia(url);
 			String content = getContentFromDocument(doc);
 			
 			//We now get the text before the first occurrence of '=='
 			int end = content.indexOf("==");
 			if (end==-1) {
 				throw new WikipediaException("Wikipediacontent for band: "+bandname+" could not be deciphered");
 			}
 			content = content.substring(0, end);
 			
 			//Now, remove everything between {{ and }}, {{}} inclusive.
 			StringBuffer sb = new StringBuffer();
 			int depth = 0;
 			for (char c : content.toCharArray()) {
 				if (c=='{') {
 					depth++;
 				} else if (c=='}') {
 					depth--;
 				} else if (depth==0) {
 					sb.append(c);
 				}
 			}
 			
 			//Once we got the first part of the text successfully, we now chop-off
 			//everything before \n, if possible.
 			content = sb.toString();
 			content = WikiContentHelper.removeComment(content);
 			content = WikiContentHelper.removeImages(content);
 			content = content.trim();
 			end = content.indexOf("\n");
 			if (end!=-1) {
 				content = content.substring(0, end);
 			}
 			
 			//Now, we have to clean-up all wiki-links that are left over.
 			content = WikiContentHelper.removeLinks(content);
 			content = WikiContentHelper.removeRefs(content);
 			content = WikiContentHelper.sanitizeQuotes(content);
 			
 			return content.trim();
 		} catch (WikipediaException e) {
 			throw new WikipediaException("No info found for band: "+bandname, e);
 		}
 	}
 	
 	/**
 	 * Get the Wikipedia URL for a Band.
 	 * @param bandname The name of the band.
 	 * @return The (external) URL, linking to the Wikipedia-entry for the Band.
 	 * The result is null if no URL could be found.
 	 */
 	public static String getExternalBandURL(String bandname) {
 		String url = null;
 		try {
 			url = NORMALURL + getBandURL(bandname);
 		} catch (WikipediaException e) {
 			log.error("Could not create the Wikipedia URL for band: " + bandname+" : "+e);
 		}
 		return url;
 	}
 	
 	
 }