Rosetta Code

WiktionaryDumps to words

Revision as of 23:50, 15 February 2021 by rosettacode>Blue Prawn (useful for spell checkers)

Make a file that can be useful with spell checkers like Ispell and Aspell.

WiktionaryDumps to words is a draft programming task. It is not yet considered ready to be promoted as a complete task, for reasons that should be found in its talk page.
NOTE
Please help addressing the issues about this task on the discussion page. If you add another language, be aware that this task may change in the future, and that you will need to update your example.
Task

Use the wiktionary dump (input) to create a file equivalent than "/usr/share/dict/spanish" (output). The input file is an XML dump of the Wiktionary that is a bz2'ed file of about 800MB. The output file should be a file similar than "/usr/share/dict/spanish" which contains one word of a given language by line in a simple text file. An example of such a file is available in Ubuntu with the package wspanish.


C

<lang C>#include <stdio.h>

  1. include <stdlib.h>
  2. include <stdbool.h>
  3. include <string.h>
  4. include <unistd.h>
  1. include <expat.h>
  2. include <pcre.h>
  1. ifdef XML_LARGE_SIZE
  2. define XML_FMT_INT_MOD "ll"
  3. else
  4. define XML_FMT_INT_MOD "l"
  5. endif
  1. ifdef XML_UNICODE_WCHAR_T
  2. define XML_FMT_STR "ls"
  3. else
  4. define XML_FMT_STR "s"
  5. endif

void reset_char_data_buffer(); void process_char_data_buffer();

static bool last_tag_is_title; static bool last_tag_is_text;

static pcre *reCompiled; static pcre_extra *pcreExtra;


void start_element(void *data, const char *element, const char **attribute) { 

   process_char_data_buffer();
   reset_char_data_buffer();

   if (strcmp("title", element) == 0) {
       last_tag_is_title = true;
   }
   if (strcmp("text", element) == 0) {
       last_tag_is_text = true;
   }

}

void end_element(void *data, const char *el) { 

   process_char_data_buffer();
   reset_char_data_buffer();

}


  1. define TITLE_BUF_SIZE (1024 * 8)

static char char_data_buffer[1024 * 64 * 8]; static char title_buffer[TITLE_BUF_SIZE]; static size_t offs; static bool overflow;


void reset_char_data_buffer(void) { 

   offs = 0;
   overflow = false;

}

// pastes parts of the node together void char_data(void *userData, const XML_Char *s, int len) { 

   if (!overflow) {
       if (len + offs >= sizeof(char_data_buffer)) {
           overflow = true;
           fprintf(stderr, "Warning: buffer overflow\n");
           fflush(stderr);
       } else {
           memcpy(char_data_buffer + offs, s, len);
           offs += len;
       }
   }

}

void try_match();

// if the element is the one we're after void process_char_data_buffer(void) { 

   if (offs > 0) {
       char_data_buffer[offs] = '\0';

       if (last_tag_is_title) {
           unsigned int n = (offs+1 > TITLE_BUF_SIZE) ? TITLE_BUF_SIZE : (offs+1);
           memcpy(title_buffer, char_data_buffer, n);
           last_tag_is_title = false;
       }
       if (last_tag_is_text) {
           try_match();
           last_tag_is_text = false;
       }
   }

}

void try_match() { 

   int subStrVec[80];
   int subStrVecLen;
   int pcreExecRet;
   subStrVecLen = sizeof(subStrVec) / sizeof(int);

   pcreExecRet = pcre_exec(
           reCompiled, pcreExtra,
           char_data_buffer, strlen(char_data_buffer),
           0, 0,
           subStrVec, subStrVecLen);

   if (pcreExecRet < 0) {
       switch (pcreExecRet) {
           case PCRE_ERROR_NOMATCH      : break;
           case PCRE_ERROR_NULL         : fprintf(stderr, "Something was null\n");                      break;
           case PCRE_ERROR_BADOPTION    : fprintf(stderr, "A bad option was passed\n");                 break;
           case PCRE_ERROR_BADMAGIC     : fprintf(stderr, "Magic number bad (compiled re corrupt?)\n"); break;
           case PCRE_ERROR_UNKNOWN_NODE : fprintf(stderr, "Something kooky in the compiled re\n");      break;
           case PCRE_ERROR_NOMEMORY     : fprintf(stderr, "Ran out of memory\n");                       break;
           default                      : fprintf(stderr, "Unknown error\n");                           break;
       }
   } else {
       puts(title_buffer);  // print the word
   }

}


  1. define BUF_SIZE 1024

int main(int argc, char *argv[]) { 

   char buffer[BUF_SIZE];
   int n;

   const char *pcreErrorStr;
   int pcreErrorOffset;
   char *aStrRegex;
   char **aLineToMatch;

   // Using PCRE

   aStrRegex = "(.*)(==French==)(.*)";  // search for French language

   reCompiled = pcre_compile(aStrRegex, PCRE_DOTALL | PCRE_UTF8, &pcreErrorStr, &pcreErrorOffset, NULL);
   if (reCompiled == NULL) {
       fprintf(stderr, "ERROR: Could not compile regex '%s': %s\n", aStrRegex, pcreErrorStr);
       exit(1);
   }

   pcreExtra = pcre_study(reCompiled, 0, &pcreErrorStr);
   if (pcreErrorStr != NULL) {
       fprintf(stderr, "ERROR: Could not study regex '%s': %s\n", aStrRegex, pcreErrorStr);
       exit(1);
   }

   // Using Expat parser

   XML_Parser parser = XML_ParserCreate(NULL);

   XML_SetElementHandler(parser, start_element, end_element);
   XML_SetCharacterDataHandler(parser, char_data);

   reset_char_data_buffer();

   while (1) {
       int done;
       int len;

       len = (int)fread(buffer, 1, BUF_SIZE, stdin);
       if (ferror(stdin)) {
           fprintf(stderr, "Read error\n");
           exit(1);
       }
       done = feof(stdin);

       if (XML_Parse(parser, buffer, len, done) == XML_STATUS_ERROR) {
           fprintf(stderr,
               "Parse error at line %" XML_FMT_INT_MOD "u:\n%" XML_FMT_STR "\n",
               XML_GetCurrentLineNumber(parser),
               XML_ErrorString(XML_GetErrorCode(parser)));
           exit(1);
       }

       if (done) break;
   }

   XML_ParserFree(parser);

   pcre_free(reCompiled);

   if (pcreExtra != NULL) {
  1. ifdef PCRE_CONFIG_JIT
       pcre_free_study(pcreExtra);
  1. else
       pcre_free(pcreExtra);
  1. endif
   }

   return 0;

}</lang>

Output:
$ gcc wikt_to_words.c -o wikt_to_words -lpcre -lexpat
$ wget --quiet https://dumps.wikimedia.org/enwiktionary/latest/enwiktionary-latest-pages-articles.xml.bz2 -O - | bzcat | \
    ./wikt_to_words
gratis
gratuit
livre
chien
pond
pies
pie
A
connotation
minute
...


Java

<lang java>import org.xml.sax.*; import org.xml.sax.helpers.DefaultHandler; import org.xml.sax.SAXException;

import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import javax.xml.parsers.ParserConfigurationException;

import java.util.regex.Pattern; import java.util.regex.Matcher;

class MyHandler extends DefaultHandler { 

   private static final String TITLE = "title";
   private static final String TEXT = "text";

   private String lastTag = "";
   private String title = "";

   @Override
   public void characters(char[] ch, int start, int length) throws SAXException {
       String regex = ".*==French==.*";
       Pattern pat = Pattern.compile(regex, Pattern.DOTALL);

       switch (lastTag) {
           case TITLE:
               title = new String(ch, start, length);
               break;
           case TEXT:
               String text = new String(ch, start, length);
               Matcher mat = pat.matcher(text);
               if (mat.matches()) {
                   System.out.println(title);
               }
               break;
       }
   }

   @Override
   public void startElement(String uri, String localName, String qName, Attributes attrs) throws SAXException {
       lastTag = qName;
   }

   @Override
   public void endElement(String uri, String localName, String qName) throws SAXException {
       lastTag = "";
   }

}

public class WiktoWords { 

   public static void main(java.lang.String[] args) {
       try {
           SAXParserFactory spFactory = SAXParserFactory.newInstance();
           SAXParser saxParser = spFactory.newSAXParser();
           MyHandler handler = new MyHandler();
           saxParser.parse(new InputSource(System.in), handler);
       } catch(Exception e) {
           System.exit(1);
       }
   }

}</lang>

Output:
$ javac WiktoWords.java
$ wget --quiet https://dumps.wikimedia.org/enwiktionary/latest/enwiktionary-latest-pages-articles.xml.bz2 -O - | bzcat | \
    java WiktoWords 
gratis
gratuit
livre
chien
pond
pies
pie
A
connotation
minute
...

OCaml

Using the library xmlm:

<lang ocaml>let () = 

 let i = Xmlm.make_input ~strip:true (`Channel stdin) in
 let title = ref "" in
 let tag_path = ref [] in
 let push_tag tag =
   tag_path := tag :: !tag_path
 in
 let pop_tag () =
   match !tag_path with [] -> ()
   | _ :: tl -> tag_path := tl
 in
 let last_tag_is tag =
   match !tag_path with [] -> false
   | hd :: _ -> hd = tag
 in
 let reg = Str.regexp_string "==French==" in
 let matches s =
   try let _ = Str.search_forward reg s 0 in true
   with Not_found -> false
 in
 while not (Xmlm.eoi i) do
   match Xmlm.input i with
   | `Dtd dtd -> ()
   | `El_start ((uri, tag_name), attrs) -> push_tag tag_name
   | `El_end -> pop_tag ()
   | `Data s ->
       if last_tag_is "title"
       then title := s;
       if last_tag_is "text"
       then begin
         if matches s
         then print_endline !title
       end
 done</lang>
Output:
wget --quiet https://dumps.wikimedia.org/enwiktionary/latest/enwiktionary-latest-pages-articles.xml.bz2 -O - | bzcat | \
  ocaml str.cma -I $(ocamlfind query xmlm) xmlm.cma to_words.ml
gratis
gratuit
livre
chien
pond
pies
pie
A
connotation
minute
...
Retrieved from "https://rosettacode.org/wiki/WiktionaryDumps_to_words?oldid=305233"