public class JSONException extends RuntimeException {
    /** Serialization ID */
    private static final long serialVersionUID = 0;

    /**
     * Constructs a JSONException with an explanatory message.
     *
     * @param message
     *            Detail about the reason for the exception.
     */
    public JSONException(final String message) {
        super(message);
    }

    /**
     * Constructs a JSONException with an explanatory message and cause.
     * 
     * @param message
     *            Detail about the reason for the exception.
     * @param cause
     *            The cause.
     */
    public JSONException(final String message, final Throwable cause) {
        super(message, cause);
    }

    /**
     * Constructs a new JSONException with the specified cause.
     * 
     * @param cause
     *            The cause.
     */
    public JSONException(final Throwable cause) {
        super(cause.getMessage(), cause);
    }

} public class JSONObject {
    /**
     * JSONObject.NULL is equivalent to the value that JavaScript calls null,
     * whilst Java's null is equivalent to the value that JavaScript calls
     * undefined.
     */
    private static final class Null {

        /**
         * There is only intended to be a single instance of the NULL object,
         * so the clone method returns itself.
         *
         * @return NULL.
         */
        @Override
        protected final Object clone() {
            return this;
        }

        /**
         * A Null object is equal to the null value and to itself.
         *
         * @param object
         *            An object to test for nullness.
         * @return true if the object parameter is the JSONObject.NULL object or
         *         null.
         */
        @Override
        public boolean equals(Object object) {
            return object == null || object == this;
        }
        /**
         * A Null object is equal to the null value and to itself.
         *
         * @return always returns 0.
         */
        @Override
        public int hashCode() {
            return 0;
        }

        /**
         * Get the "null" string value.
         *
         * @return The string "null".
         */
        @Override
        public String toString() {
            return "null";
        }
    }
    
    /**
     *  Regular Expression Pattern that matches JSON Numbers. This is primarily used for
     *  output to guarantee that we are always writing valid JSON. 
     */
    static final Pattern NUMBER_PATTERN = Pattern.compile("-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?");

    /**
     * The map where the JSONObject's properties are kept.
     */
    private final Map<String, Object> map;

    /**
     * It is sometimes more convenient and less ambiguous to have a
     * <code>NULL</code> object than to use Java's <code>null</code> value.
     * <code>JSONObject.NULL.equals(null)</code> returns <code>true</code>.
     * <code>JSONObject.NULL.toString()</code> returns <code>"null"</code>.
     */
    public static final Object NULL = new Null();

    /**
     * Construct an empty JSONObject.
     */
    public JSONObject() {
        // HashMap is used on purpose to ensure that elements are unordered by 
        // the specification.
        // JSON tends to be a portable transfer format to allows the container 
        // implementations to rearrange their items for a faster element 
        // retrieval based on associative access.
        // Therefore, an implementation mustn't rely on the order of the item.
        this.map = new HashMap<String, Object>();
    }

    /**
     * Construct an empty JSONObject.
     * 
     * @param ordered
     *            if ordered == true, then the JSONObject keeps the original order of properties
     */
    public JSONObject(boolean ordered) {
        this.map = ordered ? new LinkedHashMap<String, Object>() : new HashMap<String, Object>();
    }

    /**
     * Construct a JSONObject from a subset of another JSONObject. An array of
     * strings is used to identify the keys that should be copied. Missing keys
     * are ignored.
     *
     * @param jo
     *            A JSONObject.
     * @param names
     *            An array of strings.
     */
    public JSONObject(JSONObject jo, String ... names) {
        this(names.length);
        for (int i = 0; i < names.length; i += 1) {
            try {
                this.putOnce(names[i], jo.opt(names[i]));
            } catch (Exception ignore) {
            }
        }
    }

    /**
     * Construct a JSONObject from a JSONTokener.
     *
     * @param x
     *            A JSONTokener object containing the source string.
     * @throws JSONException
     *             If there is a syntax error in the source string or a
     *             duplicated key.
     */
    public JSONObject(JSONTokener x) throws JSONException {
        this(true);
        char c;
        String key;

        if (x.nextClean() != '{') {
            throw x.syntaxError("A JSONObject text must begin with '{'");
        }
        for (;;) {
            c = x.nextClean();
            switch (c) {
            case 0:
                throw x.syntaxError("A JSONObject text must end with '}'");
            case '}':
                return;
            default:
                x.back();
                key = x.nextValue().toString();
            }

            // The key is followed by ':'.

            c = x.nextClean();
            if (c != ':') {
                throw x.syntaxError("Expected a ':' after a key");
            }
            
            // Use syntaxError(..) to include error location
            
            if (key != null) {
                // Check if key exists
                if (this.opt(key) != null) {
                    // key already exists
                    throw x.syntaxError("Duplicate key \"" + key + "\"");
                }
                // Only add value if non-null
                Object value = x.nextValue();
                if (value!=null) {
                    this.put(key, value);
                }
            }

            // Pairs are separated by ','.

            switch (x.nextClean()) {
            case ';':
            case ',':
                if (x.nextClean() == '}') {
                    return;
                }
                x.back();
                break;
            case '}':
                return;
            default:
                throw x.syntaxError("Expected a ',' or '}'");
            }
        }
    }

    /**
     * Construct a JSONObject from a Map.
     *
     * @param m
     *            A map object that can be used to initialize the contents of
     *            the JSONObject.
     * @throws JSONException
     *            If a value in the map is non-finite number.
     * @throws NullPointerException
     *            If a key in the map is <code>null</code>
     */
    public JSONObject(Map<?, ?> m) {
        if (m == null) {
            this.map = new LinkedHashMap<String, Object>();
        } else {
            this.map = new HashMap<String, Object>(m.size());
        	for (final Entry<?, ?> e : m.entrySet()) {
        	    if(e.getKey() == null) {
        	        throw new NullPointerException("Null key.");
        	    }
                final Object value = e.getValue();
                if (value != null) {
                    this.map.put(String.valueOf(e.getKey()), wrap(value));
                }
            }
        }
    }

    /**
     * Construct a JSONObject from an Object using bean getters. It reflects on
     * all of the public methods of the object. For each of the methods with no
     * parameters and a name starting with <code>"get"</code> or
     * <code>"is"</code> followed by an uppercase letter, the method is invoked,
     * and a key and the value returned from the getter method are put into the
     * new JSONObject.
     * <p>
     * The key is formed by removing the <code>"get"</code> or <code>"is"</code>
     * prefix. If the second remaining character is not upper case, then the
     * first character is converted to lower case.
     * <p>
     * Methods that are <code>static</code>, return <code>void</code>,
     * have parameters, or are "bridge" methods, are ignored.
     * <p>
     * For example, if an object has a method named <code>"getName"</code>, and
     * if the result of calling <code>object.getName()</code> is
     * <code>"Larry Fine"</code>, then the JSONObject will contain
     * <code>"name": "Larry Fine"</code>.
     * <p>
     * The {@link JSONPropertyName} annotation can be used on a bean getter to
     * override key name used in the JSONObject. For example, using the object
     * above with the <code>getName</code> method, if we annotated it with:
     * <pre>
     * &#64;JSONPropertyName("FullName")
     * public String getName() { return this.name; }
     * </pre>
     * The resulting JSON object would contain <code>"FullName": "Larry Fine"</code>
     * <p>
     * Similarly, the {@link JSONPropertyName} annotation can be used on non-
     * <code>get</code> and <code>is</code> methods. We can also override key
     * name used in the JSONObject as seen below even though the field would normally
     * be ignored:
     * <pre>
     * &#64;JSONPropertyName("FullName")
     * public String fullName() { return this.name; }
     * </pre>
     * The resulting JSON object would contain <code>"FullName": "Larry Fine"</code>
     * <p>
     * The {@link JSONPropertyIgnore} annotation can be used to force the bean property
     * to not be serialized into JSON. If both {@link JSONPropertyIgnore} and
     * {@link JSONPropertyName} are defined on the same method, a depth comparison is
     * performed and the one closest to the concrete class being serialized is used.
     * If both annotations are at the same level, then the {@link JSONPropertyIgnore}
     * annotation takes precedent and the field is not serialized.
     * For example, the following declaration would prevent the <code>getName</code>
     * method from being serialized:
     * <pre>
     * &#64;JSONPropertyName("FullName")
     * &#64;JSONPropertyIgnore 
     * public String getName() { return this.name; }
     * </pre>
     * <p>
     * 
     * @param bean
     *            An object that has getter methods that should be used to make
     *            a JSONObject.
     */
    public JSONObject(Object bean) {
        this();
        this.populateMap(bean);
    }

    /**
     * Construct a JSONObject from an Object, using reflection to find the
     * public members. The resulting JSONObject's keys will be the strings from
     * the names array, and the values will be the field values associated with
     * those keys in the object. If a key is not found or not visible, then it
     * will not be copied into the new JSONObject.
     *
     * @param object
     *            An object that has fields that should be used to make a
     *            JSONObject.
     * @param names
     *            An array of strings, the names of the fields to be obtained
     *            from the object.
     */
    public JSONObject(Object object, String ... names) {
        this(names.length);
        Class<?> c = object.getClass();
        for (int i = 0; i < names.length; i += 1) {
            String name = names[i];
            try {
                this.putOpt(name, c.getField(name).get(object));
            } catch (Exception ignore) {
            }
        }
    }

    /**
     * Construct a JSONObject from a source JSON text string. This is the most
     * commonly used JSONObject constructor.
     *
     * @param source
     *            A string beginning with <code>{</code>&nbsp;<small>(left
     *            brace)</small> and ending with <code>}</code>
     *            &nbsp;<small>(right brace)</small>.
     * @exception JSONException
     *                If there is a syntax error in the source string or a
     *                duplicated key.
     */
    public JSONObject(String source) throws JSONException {
        this(new JSONTokener(source));
    }

    /**
     * Construct a JSONObject from a ResourceBundle.
     *
     * @param baseName
     *            The ResourceBundle base name.
     * @param locale
     *            The Locale to load the ResourceBundle for.
     * @throws JSONException
     *             If any JSONExceptions are detected.
     */
    public JSONObject(String baseName, Locale locale) throws JSONException {
        this();
        ResourceBundle bundle = ResourceBundle.getBundle(baseName, locale,
                Thread.currentThread().getContextClassLoader());

// Iterate through the keys in the bundle.

        Enumeration<String> keys = bundle.getKeys();
        while (keys.hasMoreElements()) {
            Object key = keys.nextElement();
            if (key != null) {

// Go through the path, ensuring that there is a nested JSONObject for each
// segment except the last. Add the value using the last segment's name into
// the deepest nested JSONObject.

                String[] path = ((String) key).split("\\.");
                int last = path.length - 1;
                JSONObject target = this;
                for (int i = 0; i < last; i += 1) {
                    String segment = path[i];
                    JSONObject nextTarget = target.optJSONObject(segment);
                    if (nextTarget == null) {
                        nextTarget = new JSONObject();
                        target.put(segment, nextTarget);
                    }
                    target = nextTarget;
                }
                target.put(path[last], bundle.getString((String) key));
            }
        }
    }
    
    /**
     * Constructor to specify an initial capacity of the internal map. Useful for library 
     * internal calls where we know, or at least can best guess, how big this JSONObject
     * will be.
     * 
     * @param initialCapacity initial capacity of the internal map.
     */
    protected JSONObject(int initialCapacity){
        this.map = new HashMap<String, Object>(initialCapacity);
    }

    /**
     * Accumulate values under a key. It is similar to the put method except
     * that if there is already an object stored under the key then a JSONArray
     * is stored under the key to hold all of the accumulated values. If there
     * is already a JSONArray, then the new value is appended to it. In
     * contrast, the put method replaces the previous value.
     *
     * If only one value is accumulated that is not a JSONArray, then the result
     * will be the same as using put. But if multiple values are accumulated,
     * then the result will be like append.
     *
     * @param key
     *            A key string.
     * @param value
     *            An object to be accumulated under the key.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject accumulate(String key, Object value) throws JSONException {
        testValidity(value);
        Object object = this.opt(key);
        if (object == null) {
            this.put(key,
                    value instanceof JSONArray ? new JSONArray().put(value)
                            : value);
        } else if (object instanceof JSONArray) {
            ((JSONArray) object).put(value);
        } else {
            this.put(key, new JSONArray().put(object).put(value));
        }
        return this;
    }

    /**
     * Append values to the array under a key. If the key does not exist in the
     * JSONObject, then the key is put in the JSONObject with its value being a
     * JSONArray containing the value parameter. If the key was already
     * associated with a JSONArray, then the value parameter is appended to it.
     *
     * @param key
     *            A key string.
     * @param value
     *            An object to be accumulated under the key.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number or if the current value associated with
     *             the key is not a JSONArray.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject append(String key, Object value) throws JSONException {
        testValidity(value);
        Object object = this.opt(key);
        if (object == null) {
            this.put(key, new JSONArray().put(value));
        } else if (object instanceof JSONArray) {
            this.put(key, ((JSONArray) object).put(value));
        } else {
            throw wrongValueFormatException(key, "JSONArray", null, null);
        }
        return this;
    }

    /**
     * Produce a string from a double. The string "null" will be returned if the
     * number is not finite.
     *
     * @param d
     *            A double.
     * @return A String.
     */
    public static String doubleToString(double d) {
        if (Double.isInfinite(d) || Double.isNaN(d)) {
            return "null";
        }

// Shave off trailing zeros and decimal point, if possible.

        String string = Double.toString(d);
        if (string.indexOf('.') > 0 && string.indexOf('e') < 0
                && string.indexOf('E') < 0) {
            while (string.endsWith("0")) {
                string = string.substring(0, string.length() - 1);
            }
            if (string.endsWith(".")) {
                string = string.substring(0, string.length() - 1);
            }
        }
        return string;
    }

    /**
     * Get the value object associated with a key.
     *
     * @param key
     *            A key string.
     * @return The object associated with the key.
     * @throws JSONException
     *             if the key is not found.
     */
    public Object get(String key) throws JSONException {
        if (key == null) {
            throw new JSONException("Null key.");
        }
        Object object = this.opt(key);
        if (object == null) {
            throw new JSONException("JSONObject[" + quote(key) + "] not found.");
        }
        return object;
    }

    /**
     * Get the enum value associated with a key.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *           The type of enum to retrieve.
     * @param key
     *           A key string.
     * @return The enum value associated with the key
     * @throws JSONException
     *             if the key is not found or if the value cannot be converted
     *             to an enum.
     */
    public <E extends Enum<E>> E getEnum(Class<E> clazz, String key) throws JSONException {
        E val = optEnum(clazz, key);
        if(val==null) {
            // JSONException should really take a throwable argument.
            // If it did, I would re-implement this with the Enum.valueOf
            // method and place any thrown exception in the JSONException
            throw wrongValueFormatException(key, "enum of type " + quote(clazz.getSimpleName()), null);
        }
        return val;
    }

    /**
     * Get the boolean value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The truth.
     * @throws JSONException
     *             if the value is not a Boolean or the String "true" or
     *             "false".
     */
    public boolean getBoolean(String key) throws JSONException {
        Object object = this.get(key);
        if (object.equals(Boolean.FALSE)
                || (object instanceof String && ((String) object)
                        .equalsIgnoreCase("false"))) {
            return false;
        } else if (object.equals(Boolean.TRUE)
                || (object instanceof String && ((String) object)
                        .equalsIgnoreCase("true"))) {
            return true;
        }
        throw wrongValueFormatException(key, "Boolean", null);
    }

    /**
     * Get the BigInteger value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value cannot 
     *             be converted to BigInteger.
     */
    public BigInteger getBigInteger(String key) throws JSONException {
        Object object = this.get(key);
        BigInteger ret = objectToBigInteger(object, null);
        if (ret != null) {
            return ret;
        }
        throw wrongValueFormatException(key, "BigInteger", object, null);
    }

    /**
     * Get the BigDecimal value associated with a key. If the value is float or
     * double, the the {@link BigDecimal#BigDecimal(double)} constructor will
     * be used. See notes on the constructor for conversion issues that may
     * arise.
     *
     * @param key
     *            A key string.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value
     *             cannot be converted to BigDecimal.
     */
    public BigDecimal getBigDecimal(String key) throws JSONException {
        Object object = this.get(key);
        BigDecimal ret = objectToBigDecimal(object, null);
        if (ret != null) {
            return ret;
        }
        throw wrongValueFormatException(key, "BigDecimal", object, null);
    }

    /**
     * Get the double value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value is not a Number
     *             object and cannot be converted to a number.
     */
    public double getDouble(String key) throws JSONException {
        final Object object = this.get(key);
        if(object instanceof Number) {
            return ((Number)object).doubleValue();
        }
        try {
            return Double.parseDouble(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(key, "double", e);
        }
    }

    /**
     * Get the float value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value is not a Number
     *             object and cannot be converted to a number.
     */
    public float getFloat(String key) throws JSONException {
        final Object object = this.get(key);
        if(object instanceof Number) {
            return ((Number)object).floatValue();
        }
        try {
            return Float.parseFloat(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(key, "float", e);
        }
    }

    /**
     * Get the Number value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value is not a Number
     *             object and cannot be converted to a number.
     */
    public Number getNumber(String key) throws JSONException {
        Object object = this.get(key);
        try {
            if (object instanceof Number) {
                return (Number)object;
            }
            return stringToNumber(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(key, "number", e);
        }
    }

    /**
     * Get the int value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The integer value.
     * @throws JSONException
     *             if the key is not found or if the value cannot be converted
     *             to an integer.
     */
    public int getInt(String key) throws JSONException {
        final Object object = this.get(key);
        if(object instanceof Number) {
            return ((Number)object).intValue();
        }
        try {
            return Integer.parseInt(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(key, "int", e);
        }
    }

    /**
     * Get the JSONArray value associated with a key.
     *
     * @param key
     *            A key string.
     * @return A JSONArray which is the value.
     * @throws JSONException
     *             if the key is not found or if the value is not a JSONArray.
     */
    public JSONArray getJSONArray(String key) throws JSONException {
        Object object = this.get(key);
        if (object instanceof JSONArray) {
            return (JSONArray) object;
        }
        throw wrongValueFormatException(key, "JSONArray", null);
    }

    /**
     * Get the JSONObject value associated with a key.
     *
     * @param key
     *            A key string.
     * @return A JSONObject which is the value.
     * @throws JSONException
     *             if the key is not found or if the value is not a JSONObject.
     */
    public JSONObject getJSONObject(String key) throws JSONException {
        Object object = this.get(key);
        if (object instanceof JSONObject) {
            return (JSONObject) object;
        }
        throw wrongValueFormatException(key, "JSONObject", null);
    }

    /**
     * Get the long value associated with a key.
     *
     * @param key
     *            A key string.
     * @return The long value.
     * @throws JSONException
     *             if the key is not found or if the value cannot be converted
     *             to a long.
     */
    public long getLong(String key) throws JSONException {
        final Object object = this.get(key);
        if(object instanceof Number) {
            return ((Number)object).longValue();
        }
        try {
            return Long.parseLong(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(key, "long", e);
        }
    }

    /**
     * Get an array of field names from a JSONObject.
     *
     * @param jo
     *            JSON object
     * @return An array of field names, or null if there are no names.
     */
    public static String[] getNames(JSONObject jo) {
        if (jo.isEmpty()) {
            return null;
        }
        return jo.keySet().toArray(new String[jo.length()]);
    }

    /**
     * Get an array of public field names from an Object.
     *
     * @param object
     *            object to read
     * @return An array of field names, or null if there are no names.
     */
    public static String[] getNames(Object object) {
        if (object == null) {
            return null;
        }
        Class<?> klass = object.getClass();
        Field[] fields = klass.getFields();
        int length = fields.length;
        if (length == 0) {
            return null;
        }
        String[] names = new String[length];
        for (int i = 0; i < length; i += 1) {
            names[i] = fields[i].getName();
        }
        return names;
    }

    /**
     * Get the string associated with a key.
     *
     * @param key
     *            A key string.
     * @return A string which is the value.
     * @throws JSONException
     *             if there is no string value for the key.
     */
    public String getString(String key) throws JSONException {
        Object object = this.get(key);
        if (object instanceof String) {
            return (String) object;
        }
        throw wrongValueFormatException(key, "string", null);
    }

    /**
     * Determine if the JSONObject contains a specific key.
     *
     * @param key
     *            A key string.
     * @return true if the key exists in the JSONObject.
     */
    public boolean has(String key) {
        return this.map.containsKey(key);
    }

    /**
     * Increment a property of a JSONObject. If there is no such property,
     * create one with a value of 1 (Integer). If there is such a property, and if it is
     * an Integer, Long, Double, Float, BigInteger, or BigDecimal then add one to it.
     * No overflow bounds checking is performed, so callers should initialize the key
     * prior to this call with an appropriate type that can handle the maximum expected
     * value.
     *
     * @param key
     *            A key string.
     * @return this.
     * @throws JSONException
     *             If there is already a property with this name that is not an
     *             Integer, Long, Double, or Float.
     */
    public JSONObject increment(String key) throws JSONException {
        Object value = this.opt(key);
        if (value == null) {
            this.put(key, 1);
        } else if (value instanceof Integer) {
            this.put(key, ((Integer) value).intValue() + 1);
        } else if (value instanceof Long) {
            this.put(key, ((Long) value).longValue() + 1L);
        } else if (value instanceof BigInteger) {
            this.put(key, ((BigInteger)value).add(BigInteger.ONE));
        } else if (value instanceof Float) {
            this.put(key, ((Float) value).floatValue() + 1.0f);
        } else if (value instanceof Double) {
            this.put(key, ((Double) value).doubleValue() + 1.0d);
        } else if (value instanceof BigDecimal) {
            this.put(key, ((BigDecimal)value).add(BigDecimal.ONE));
        } else {
            throw new JSONException("Unable to increment [" + quote(key) + "].");
        }
        return this;
    }

    /**
     * Determine if the value associated with the key is <code>null</code> or if there is no
     * value.
     *
     * @param key
     *            A key string.
     * @return true if there is no value associated with the key or if the value
     *        is the JSONObject.NULL object.
     */
    public boolean isNull(String key) {
        return JSONObject.NULL.equals(this.opt(key));
    }

    /**
     * Get an enumeration of the keys of the JSONObject. Modifying this key Set will also
     * modify the JSONObject. Use with caution.
     *
     * @see Set#iterator()
     * 
     * @return An iterator of the keys.
     */
    public Iterator<String> keys() {
        return this.keySet().iterator();
    }

    /**
     * Get a set of keys of the JSONObject. Modifying this key Set will also modify the
     * JSONObject. Use with caution.
     *
     * @see Map#keySet()
     *
     * @return A keySet.
     */
    public Set<String> keySet() {
        return this.map.keySet();
    }

    /**
     * Get a set of entries of the JSONObject. These are raw values and may not
     * match what is returned by the JSONObject get* and opt* functions. Modifying 
     * the returned EntrySet or the Entry objects contained therein will modify the
     * backing JSONObject. This does not return a clone or a read-only view.
     * 
     * Use with caution.
     *
     * @see Map#entrySet()
     *
     * @return An Entry Set
     */
    protected Set<Entry<String, Object>> entrySet() {
        return this.map.entrySet();
    }

    /**
     * Get the number of keys stored in the JSONObject.
     *
     * @return The number of keys in the JSONObject.
     */
    public int length() {
        return this.map.size();
    }

    /**
     * Check if JSONObject is empty.
     *
     * @return true if JSONObject is empty, otherwise false.
     */
    public boolean isEmpty() {
        return this.map.isEmpty();
    }

    /**
     * Produce a JSONArray containing the names of the elements of this
     * JSONObject.
     *
     * @return A JSONArray containing the key strings, or null if the JSONObject
     *        is empty.
     */
    public JSONArray names() {
    	if(this.map.isEmpty()) {
    		return null;
    	}
        return new JSONArray(this.map.keySet());
    }

    /**
     * Produce a string from a Number.
     *
     * @param number
     *            A Number
     * @return A String.
     * @throws JSONException
     *             If n is a non-finite number.
     */
    public static String numberToString(Number number) throws JSONException {
        if (number == null) {
            throw new JSONException("Null pointer");
        }
        testValidity(number);

        // Shave off trailing zeros and decimal point, if possible.

        String string = number.toString();
        if (string.indexOf('.') > 0 && string.indexOf('e') < 0
                && string.indexOf('E') < 0) {
            while (string.endsWith("0")) {
                string = string.substring(0, string.length() - 1);
            }
            if (string.endsWith(".")) {
                string = string.substring(0, string.length() - 1);
            }
        }
        return string;
    }

    /**
     * Get an optional value associated with a key.
     *
     * @param key
     *            A key string.
     * @return An object which is the value, or null if there is no value.
     */
    public Object opt(String key) {
        return key == null ? null : this.map.get(key);
    }

    /**
     * Get the enum value associated with a key.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *            The type of enum to retrieve.
     * @param key
     *            A key string.
     * @return The enum value associated with the key or null if not found
     */
    public <E extends Enum<E>> E optEnum(Class<E> clazz, String key) {
        return this.optEnum(clazz, key, null);
    }

    /**
     * Get the enum value associated with a key.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *            The type of enum to retrieve.
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default in case the value is not found
     * @return The enum value associated with the key or defaultValue
     *            if the value is not found or cannot be assigned to <code>clazz</code>
     */
    public <E extends Enum<E>> E optEnum(Class<E> clazz, String key, E defaultValue) {
        try {
            Object val = this.opt(key);
            if (NULL.equals(val)) {
                return defaultValue;
            }
            if (clazz.isAssignableFrom(val.getClass())) {
                // we just checked it!
                @SuppressWarnings("unchecked")
                E myE = (E) val;
                return myE;
            }
            return Enum.valueOf(clazz, val.toString());
        } catch (IllegalArgumentException e) {
            return defaultValue;
        } catch (NullPointerException e) {
            return defaultValue;
        }
    }

    /**
     * Get an optional boolean associated with a key. It returns false if there
     * is no such key, or if the value is not Boolean.TRUE or the String "true".
     *
     * @param key
     *            A key string.
     * @return The truth.
     */
    public boolean optBoolean(String key) {
        return this.optBoolean(key, false);
    }

    /**
     * Get an optional boolean associated with a key. It returns the
     * defaultValue if there is no such key, or if it is not a Boolean or the
     * String "true" or "false" (case insensitive).
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return The truth.
     */
    public boolean optBoolean(String key, boolean defaultValue) {
        Object val = this.opt(key);
        if (NULL.equals(val)) {
            return defaultValue;
        }
        if (val instanceof Boolean){
            return ((Boolean) val).booleanValue();
        }
        try {
            // we'll use the get anyway because it does string conversion.
            return this.getBoolean(key);
        } catch (Exception e) {
            return defaultValue;
        }
    }

    /**
     * Get an optional BigDecimal associated with a key, or the defaultValue if
     * there is no such key or if its value is not a number. If the value is a
     * string, an attempt will be made to evaluate it as a number. If the value
     * is float or double, then the {@link BigDecimal#BigDecimal(double)}
     * constructor will be used. See notes on the constructor for conversion
     * issues that may arise.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public BigDecimal optBigDecimal(String key, BigDecimal defaultValue) {
        Object val = this.opt(key);
        return objectToBigDecimal(val, defaultValue);
    }

    /**
     * @param val value to convert
     * @param defaultValue default value to return is the conversion doesn't work or is null.
     * @return BigDecimal conversion of the original value, or the defaultValue if unable
     *          to convert. 
     */
    static BigDecimal objectToBigDecimal(Object val, BigDecimal defaultValue) {
        if (NULL.equals(val)) {
            return defaultValue;
        }
        if (val instanceof BigDecimal){
            return (BigDecimal) val;
        }
        if (val instanceof BigInteger){
            return new BigDecimal((BigInteger) val);
        }
        if (val instanceof Double || val instanceof Float){
            final double d = ((Number) val).doubleValue();
            if(Double.isNaN(d)) {
                return defaultValue;
            }
            return new BigDecimal(((Number) val).doubleValue());
        }
        if (val instanceof Long || val instanceof Integer
                || val instanceof Short || val instanceof Byte){
            return new BigDecimal(((Number) val).longValue());
        }
        // don't check if it's a string in case of unchecked Number subclasses
        try {
            return new BigDecimal(val.toString());
        } catch (Exception e) {
            return defaultValue;
        }
    }

    /**
     * Get an optional BigInteger associated with a key, or the defaultValue if
     * there is no such key or if its value is not a number. If the value is a
     * string, an attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public BigInteger optBigInteger(String key, BigInteger defaultValue) {
        Object val = this.opt(key);
        return objectToBigInteger(val, defaultValue);
    }

    /**
     * @param val value to convert
     * @param defaultValue default value to return is the conversion doesn't work or is null.
     * @return BigInteger conversion of the original value, or the defaultValue if unable
     *          to convert. 
     */
    static BigInteger objectToBigInteger(Object val, BigInteger defaultValue) {
        if (NULL.equals(val)) {
            return defaultValue;
        }
        if (val instanceof BigInteger){
            return (BigInteger) val;
        }
        if (val instanceof BigDecimal){
            return ((BigDecimal) val).toBigInteger();
        }
        if (val instanceof Double || val instanceof Float){
            final double d = ((Number) val).doubleValue();
            if(Double.isNaN(d)) {
                return defaultValue;
            }
            return new BigDecimal(d).toBigInteger();
        }
        if (val instanceof Long || val instanceof Integer
                || val instanceof Short || val instanceof Byte){
            return BigInteger.valueOf(((Number) val).longValue());
        }
        // don't check if it's a string in case of unchecked Number subclasses
        try {
            // the other opt functions handle implicit conversions, i.e. 
            // jo.put("double",1.1d);
            // jo.optInt("double"); -- will return 1, not an error
            // this conversion to BigDecimal then to BigInteger is to maintain
            // that type cast support that may truncate the decimal.
            final String valStr = val.toString();
            if(isDecimalNotation(valStr)) {
                return new BigDecimal(valStr).toBigInteger();
            }
            return new BigInteger(valStr);
        } catch (Exception e) {
            return defaultValue;
        }
    }

    /**
     * Get an optional double associated with a key, or NaN if there is no such
     * key or if its value is not a number. If the value is a string, an attempt
     * will be made to evaluate it as a number.
     *
     * @param key
     *            A string which is the key.
     * @return An object which is the value.
     */
    public double optDouble(String key) {
        return this.optDouble(key, Double.NaN);
    }

    /**
     * Get an optional double associated with a key, or the defaultValue if
     * there is no such key or if its value is not a number. If the value is a
     * string, an attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public double optDouble(String key, double defaultValue) {
        Number val = this.optNumber(key);
        if (val == null) {
            return defaultValue;
        }
        final double doubleValue = val.doubleValue();
        // if (Double.isNaN(doubleValue) || Double.isInfinite(doubleValue)) {
        // return defaultValue;
        // }
        return doubleValue;
    }

    /**
     * Get the optional double value associated with an index. NaN is returned
     * if there is no value for the index, or if the value is not a number and
     * cannot be converted to a number.
     *
     * @param key
     *            A key string.
     * @return The value.
     */
    public float optFloat(String key) {
        return this.optFloat(key, Float.NaN);
    }

    /**
     * Get the optional double value associated with an index. The defaultValue
     * is returned if there is no value for the index, or if the value is not a
     * number and cannot be converted to a number.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public float optFloat(String key, float defaultValue) {
        Number val = this.optNumber(key);
        if (val == null) {
            return defaultValue;
        }
        final float floatValue = val.floatValue();
        // if (Float.isNaN(floatValue) || Float.isInfinite(floatValue)) {
        // return defaultValue;
        // }
        return floatValue;
    }

    /**
     * Get an optional int value associated with a key, or zero if there is no
     * such key or if the value is not a number. If the value is a string, an
     * attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @return An object which is the value.
     */
    public int optInt(String key) {
        return this.optInt(key, 0);
    }

    /**
     * Get an optional int value associated with a key, or the default if there
     * is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public int optInt(String key, int defaultValue) {
        final Number val = this.optNumber(key, null);
        if (val == null) {
            return defaultValue;
        }
        return val.intValue();
    }

    /**
     * Get an optional JSONArray associated with a key. It returns null if there
     * is no such key, or if its value is not a JSONArray.
     *
     * @param key
     *            A key string.
     * @return A JSONArray which is the value.
     */
    public JSONArray optJSONArray(String key) {
        Object o = this.opt(key);
        return o instanceof JSONArray ? (JSONArray) o : null;
    }

    /**
     * Get an optional JSONObject associated with a key. It returns null if
     * there is no such key, or if its value is not a JSONObject.
     *
     * @param key
     *            A key string.
     * @return A JSONObject which is the value.
     */
    public JSONObject optJSONObject(String key) {
        Object object = this.opt(key);
        return object instanceof JSONObject ? (JSONObject) object : null;
    }

    /**
     * Get an optional long value associated with a key, or zero if there is no
     * such key or if the value is not a number. If the value is a string, an
     * attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @return An object which is the value.
     */
    public long optLong(String key) {
        return this.optLong(key, 0);
    }

    /**
     * Get an optional long value associated with a key, or the default if there
     * is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public long optLong(String key, long defaultValue) {
        final Number val = this.optNumber(key, null);
        if (val == null) {
            return defaultValue;
        }
        
        return val.longValue();
    }
    
    /**
     * Get an optional {@link Number} value associated with a key, or <code>null</code>
     * if there is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number ({@link BigDecimal}). This method
     * would be used in cases where type coercion of the number value is unwanted.
     *
     * @param key
     *            A key string.
     * @return An object which is the value.
     */
    public Number optNumber(String key) {
        return this.optNumber(key, null);
    }

    /**
     * Get an optional {@link Number} value associated with a key, or the default if there
     * is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number. This method
     * would be used in cases where type coercion of the number value is unwanted.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public Number optNumber(String key, Number defaultValue) {
        Object val = this.opt(key);
        if (NULL.equals(val)) {
            return defaultValue;
        }
        if (val instanceof Number){
            return (Number) val;
        }
        
        try {
            return stringToNumber(val.toString());
        } catch (Exception e) {
            return defaultValue;
        }
    }
    
    /**
     * Get an optional string associated with a key. It returns an empty string
     * if there is no such key. If the value is not a string and is not null,
     * then it is converted to a string.
     *
     * @param key
     *            A key string.
     * @return A string which is the value.
     */
    public String optString(String key) {
        return this.optString(key, "");
    }

    /**
     * Get an optional string associated with a key. It returns the defaultValue
     * if there is no such key.
     *
     * @param key
     *            A key string.
     * @param defaultValue
     *            The default.
     * @return A string which is the value.
     */
    public String optString(String key, String defaultValue) {
        Object object = this.opt(key);
        return NULL.equals(object) ? defaultValue : object.toString();
    }

    /**
     * Populates the internal map of the JSONObject with the bean properties. The
     * bean can not be recursive.
     *
     * @see JSONObject#JSONObject(Object)
     *
     * @param bean
     *            the bean
     */
    private void populateMap(Object bean) {
        Class<?> klass = bean.getClass();

        // If klass is a System class then set includeSuperClass to false.

        boolean includeSuperClass = klass.getClassLoader() != null;

        Method[] methods = includeSuperClass ? klass.getMethods() : klass.getDeclaredMethods();
        for (final Method method : methods) {
            final int modifiers = method.getModifiers();
            if (Modifier.isPublic(modifiers)
                    && !Modifier.isStatic(modifiers)
                    && method.getParameterTypes().length == 0
                    && !method.isBridge()
                    && method.getReturnType() != Void.TYPE
                    && isValidMethodName(method.getName())) {
                final String key = getKeyNameFromMethod(method);
                if (key != null && !key.isEmpty()) {
                    try {
                        final Object result = method.invoke(bean);
                        if (result != null) {
                            this.map.put(key, wrap(result));
                            // we don't use the result anywhere outside of wrap
                            // if it's a resource we should be sure to close it
                            // after calling toString
                            if (result instanceof Closeable) {
                                try {
                                    ((Closeable) result).close();
                                } catch (IOException ignore) {
                                }
                            }
                        }
                    } catch (IllegalAccessException ignore) {
                    } catch (IllegalArgumentException ignore) {
                    } catch (InvocationTargetException ignore) {
                    }
                }
            }
        }
    }

    private static boolean isValidMethodName(String name) {
        return !"getClass".equals(name) && !"getDeclaringClass".equals(name);
    }

    private static String getKeyNameFromMethod(Method method) {
        final int ignoreDepth = getAnnotationDepth(method, JSONPropertyIgnore.class);
        if (ignoreDepth > 0) {
            final int forcedNameDepth = getAnnotationDepth(method, JSONPropertyName.class);
            if (forcedNameDepth < 0 || ignoreDepth <= forcedNameDepth) {
                // the hierarchy asked to ignore, and the nearest name override
                // was higher or non-existent
                return null;
            }
        }
        JSONPropertyName annotation = getAnnotation(method, JSONPropertyName.class);
        if (annotation != null && annotation.value() != null && !annotation.value().isEmpty()) {
            return annotation.value();
        }
        String key;
        final String name = method.getName();
        if (name.startsWith("get") && name.length() > 3) {
            key = name.substring(3);
        } else if (name.startsWith("is") && name.length() > 2) {
            key = name.substring(2);
        } else {
            return null;
        }
        // if the first letter in the key is not uppercase, then skip.
        // This is to maintain backwards compatibility before PR406
        // (https://github.com/stleary/JSON-java/pull/406/)
        if (Character.isLowerCase(key.charAt(0))) {
            return null;
        }
        if (key.length() == 1) {
            key = key.toLowerCase(Locale.ROOT);
        } else if (!Character.isUpperCase(key.charAt(1))) {
            key = key.substring(0, 1).toLowerCase(Locale.ROOT) + key.substring(1);
        }
        return key;
    }

    /**
     * Searches the class hierarchy to see if the method or it's super
     * implementations and interfaces has the annotation.
     *
     * @param <A>
     *            type of the annotation
     *
     * @param m
     *            method to check
     * @param annotationClass
     *            annotation to look for
     * @return the {@link Annotation} if the annotation exists on the current method
     *         or one of it's super class definitions
     */
    private static <A extends Annotation> A getAnnotation(final Method m, final Class<A> annotationClass) {
        // if we have invalid data the result is null
        if (m == null || annotationClass == null) {
            return null;
        }

        if (m.isAnnotationPresent(annotationClass)) {
            return m.getAnnotation(annotationClass);
        }

        // if we've already reached the Object class, return null;
        Class<?> c = m.getDeclaringClass();
        if (c.getSuperclass() == null) {
            return null;
        }

        // check directly implemented interfaces for the method being checked
        for (Class<?> i : c.getInterfaces()) {
            try {
                Method im = i.getMethod(m.getName(), m.getParameterTypes());
                return getAnnotation(im, annotationClass);
            } catch (final SecurityException ex) {
                continue;
            } catch (final NoSuchMethodException ex) {
                continue;
            }
        }

        try {
            return getAnnotation(
                    c.getSuperclass().getMethod(m.getName(), m.getParameterTypes()),
                    annotationClass);
        } catch (final SecurityException ex) {
            return null;
        } catch (final NoSuchMethodException ex) {
            return null;
        }
    }

    /**
     * Searches the class hierarchy to see if the method or it's super
     * implementations and interfaces has the annotation. Returns the depth of the
     * annotation in the hierarchy.
     *
     * @param <A>
     *            type of the annotation
     *
     * @param m
     *            method to check
     * @param annotationClass
     *            annotation to look for
     * @return Depth of the annotation or -1 if the annotation is not on the method.
     */
    private static int getAnnotationDepth(final Method m, final Class<? extends Annotation> annotationClass) {
        // if we have invalid data the result is -1
        if (m == null || annotationClass == null) {
            return -1;
        }

        if (m.isAnnotationPresent(annotationClass)) {
            return 1;
        }

        // if we've already reached the Object class, return -1;
        Class<?> c = m.getDeclaringClass();
        if (c.getSuperclass() == null) {
            return -1;
        }

        // check directly implemented interfaces for the method being checked
        for (Class<?> i : c.getInterfaces()) {
            try {
                Method im = i.getMethod(m.getName(), m.getParameterTypes());
                int d = getAnnotationDepth(im, annotationClass);
                if (d > 0) {
                    // since the annotation was on the interface, add 1
                    return d + 1;
                }
            } catch (final SecurityException ex) {
                continue;
            } catch (final NoSuchMethodException ex) {
                continue;
            }
        }

        try {
            int d = getAnnotationDepth(
                    c.getSuperclass().getMethod(m.getName(), m.getParameterTypes()),
                    annotationClass);
            if (d > 0) {
                // since the annotation was on the superclass, add 1
                return d + 1;
            }
            return -1;
        } catch (final SecurityException ex) {
            return -1;
        } catch (final NoSuchMethodException ex) {
            return -1;
        }
    }

    /**
     * Put a key/boolean pair in the JSONObject.
     *
     * @param key
     *            A key string.
     * @param value
     *            A boolean which is the value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, boolean value) throws JSONException {
        return this.put(key, value ? Boolean.TRUE : Boolean.FALSE);
    }

    /**
     * Put a key/value pair in the JSONObject, where the value will be a
     * JSONArray which is produced from a Collection.
     *
     * @param key
     *            A key string.
     * @param value
     *            A Collection value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, Collection<?> value) throws JSONException {
        return this.put(key, new JSONArray(value));
    }

    /**
     * Put a key/double pair in the JSONObject.
     *
     * @param key
     *            A key string.
     * @param value
     *            A double which is the value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, double value) throws JSONException {
        return this.put(key, Double.valueOf(value));
    }
    
    /**
     * Put a key/float pair in the JSONObject.
     *
     * @param key
     *            A key string.
     * @param value
     *            A float which is the value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, float value) throws JSONException {
        return this.put(key, Float.valueOf(value));
    }

    /**
     * Put a key/int pair in the JSONObject.
     *
     * @param key
     *            A key string.
     * @param value
     *            An int which is the value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, int value) throws JSONException {
        return this.put(key, Integer.valueOf(value));
    }

    /**
     * Put a key/long pair in the JSONObject.
     *
     * @param key
     *            A key string.
     * @param value
     *            A long which is the value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, long value) throws JSONException {
        return this.put(key, Long.valueOf(value));
    }

    /**
     * Put a key/value pair in the JSONObject, where the value will be a
     * JSONObject which is produced from a Map.
     *
     * @param key
     *            A key string.
     * @param value
     *            A Map value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, Map<?, ?> value) throws JSONException {
        return this.put(key, new JSONObject(value));
    }

    /**
     * Put a key/value pair in the JSONObject. If the value is <code>null</code>, then the
     * key will be removed from the JSONObject if it is present.
     *
     * @param key
     *            A key string.
     * @param value
     *            An object which is the value. It should be of one of these
     *            types: Boolean, Double, Integer, JSONArray, JSONObject, Long,
     *            String, or the JSONObject.NULL object.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     * @throws NullPointerException
     *            If the key is <code>null</code>.
     */
    public JSONObject put(String key, Object value) throws JSONException {
        if (key == null) {
            throw new NullPointerException("Null key.");
        }
        if (value != null) {
            testValidity(value);
            this.map.put(key, value);
        } else {
            this.remove(key);
        }
        return this;
    }

    public void putAll(JSONObject other) {
        this.map.putAll(other.map);
    }

    /**
     * Put a key/value pair in the JSONObject, but only if the key and the value
     * are both non-null, and only if there is not already a member with that
     * name.
     *
     * @param key
     *            key to insert into
     * @param value
     *            value to insert
     * @return this.
     * @throws JSONException
     *             if the key is a duplicate
     */
    public JSONObject putOnce(String key, Object value) throws JSONException {
        if (key != null && value != null) {
            if (this.opt(key) != null) {
                throw new JSONException("Duplicate key \"" + key + "\"");
            }
            return this.put(key, value);
        }
        return this;
    }

    /**
     * Put a key/value pair in the JSONObject, but only if the key and the value
     * are both non-null.
     *
     * @param key
     *            A key string.
     * @param value
     *            An object which is the value. It should be of one of these
     *            types: Boolean, Double, Integer, JSONArray, JSONObject, Long,
     *            String, or the JSONObject.NULL object.
     * @return this.
     * @throws JSONException
     *             If the value is a non-finite number.
     */
    public JSONObject putOpt(String key, Object value) throws JSONException {
        if (key != null && value != null) {
            return this.put(key, value);
        }
        return this;
    }

    /**
     * Creates a JSONPointer using an initialization string and tries to 
     * match it to an item within this JSONObject. For example, given a
     * JSONObject initialized with this document:
     * <pre>
     * {
     *     "a":{"b":"c"}
     * }
     * </pre>
     * and this JSONPointer string: 
     * <pre>
     * "/a/b"
     * </pre>
     * Then this method will return the String "c".
     * A JSONPointerException may be thrown from code called by this method.
     *   
     * @param jsonPointer string that can be used to create a JSONPointer
     * @return the item matched by the JSONPointer, otherwise null
     */
    public Object query(String jsonPointer) {
        return query(new JSONPointer(jsonPointer));
    }
    /**
     * Uses a user initialized JSONPointer  and tries to 
     * match it to an item within this JSONObject. For example, given a
     * JSONObject initialized with this document:
     * <pre>
     * {
     *     "a":{"b":"c"}
     * }
     * </pre>
     * and this JSONPointer: 
     * <pre>
     * "/a/b"
     * </pre>
     * Then this method will return the String "c".
     * A JSONPointerException may be thrown from code called by this method.
     *   
     * @param jsonPointer string that can be used to create a JSONPointer
     * @return the item matched by the JSONPointer, otherwise null
     */
    public Object query(JSONPointer jsonPointer) {
        return jsonPointer.queryFrom(this);
    }
    
    /**
     * Queries and returns a value from this object using {@code jsonPointer}, or
     * returns null if the query fails due to a missing key.
     * 
     * @param jsonPointer the string representation of the JSON pointer
     * @return the queried value or {@code null}
     * @throws IllegalArgumentException if {@code jsonPointer} has invalid syntax
     */
    public Object optQuery(String jsonPointer) {
    	return optQuery(new JSONPointer(jsonPointer));
    }
    
    /**
     * Queries and returns a value from this object using {@code jsonPointer}, or
     * returns null if the query fails due to a missing key.
     * 
     * @param jsonPointer The JSON pointer
     * @return the queried value or {@code null}
     * @throws IllegalArgumentException if {@code jsonPointer} has invalid syntax
     */
    public Object optQuery(JSONPointer jsonPointer) {
        try {
            return jsonPointer.queryFrom(this);
        } catch (JSONPointerException e) {
            return null;
        }
    }

    /**
     * Produce a string in double quotes with backslash sequences in all the
     * right places. A backslash will be inserted within &lt;/, producing
     * &lt;\/, allowing JSON text to be delivered in HTML. In JSON text, a
     * string cannot contain a control character or an unescaped quote or
     * backslash.
     *
     * @param string
     *            A String
     * @return A String correctly formatted for insertion in a JSON text.
     */
    public static String quote(String string) {
        StringWriter sw = new StringWriter();
        synchronized (sw.getBuffer()) {
            try {
                return quote(string, sw).toString();
            } catch (IOException ignored) {
                // will never happen - we are writing to a string writer
                return "";
            }
        }
    }

    public static Writer quote(String string, Writer w) throws IOException {
        if (string == null || string.isEmpty()) {
            w.write("\"\"");
            return w;
        }

        char b;
        char c = 0;
        String hhhh;
        int i;
        int len = string.length();

        w.write('"');
        for (i = 0; i < len; i += 1) {
            b = c;
            c = string.charAt(i);
            switch (c) {
            case '\\':
            case '"':
                w.write('\\');
                w.write(c);
                break;
            case '/':
                if (b == '<') {
                    w.write('\\');
                }
                w.write(c);
                break;
            case '\b':
                w.write("\\b");
                break;
            case '\t':
                w.write("\\t");
                break;
            case '\n':
                w.write("\\n");
                break;
            case '\f':
                w.write("\\f");
                break;
            case '\r':
                w.write("\\r");
                break;
            default:
                if (c < ' ' || (c >= '\u0080' && c < '\u00a0')
                        || (c >= '\u2000' && c < '\u2100')) {
                    w.write("\\u");
                    hhhh = Integer.toHexString(c);
                    w.write("0000", 0, 4 - hhhh.length());
                    w.write(hhhh);
                } else {
                    w.write(c);
                }
            }
        }
        w.write('"');
        return w;
    }

    /**
     * Remove a name and its value, if present.
     *
     * @param key
     *            The name to be removed.
     * @return The value that was associated with the name, or null if there was
     *         no value.
     */
    public Object remove(String key) {
        return this.map.remove(key);
    }

    /**
     * Determine if two JSONObjects are similar.
     * They must contain the same set of names which must be associated with
     * similar values.
     *
     * @param other The other JSONObject
     * @return true if they are equal
     */
    public boolean similar(Object other) {
        try {
            if (!(other instanceof JSONObject)) {
                return false;
            }
            if (!this.keySet().equals(((JSONObject)other).keySet())) {
                return false;
            }
            for (final Entry<String,?> entry : this.entrySet()) {
                String name = entry.getKey();
                Object valueThis = entry.getValue();
                Object valueOther = ((JSONObject)other).get(name);
                if(valueThis == valueOther) {
                	continue;
                }
                if(valueThis == null) {
                	return false;
                }
                if (valueThis instanceof JSONObject) {
                    if (!((JSONObject)valueThis).similar(valueOther)) {
                        return false;
                    }
                } else if (valueThis instanceof JSONArray) {
                    if (!((JSONArray)valueThis).similar(valueOther)) {
                        return false;
                    }
                } else if (!valueThis.equals(valueOther)) {
                    return false;
                }
            }
            return true;
        } catch (Throwable exception) {
            return false;
        }
    }
    
    /**
     * Tests if the value should be tried as a decimal. It makes no test if there are actual digits.
     * 
     * @param val value to test
     * @return true if the string is "-0" or if it contains '.', 'e', or 'E', false otherwise.
     */
    protected static boolean isDecimalNotation(final String val) {
        return val.indexOf('.') > -1 || val.indexOf('e') > -1
                || val.indexOf('E') > -1 || "-0".equals(val);
    }
    
    /**
     * Converts a string to a number using the narrowest possible type. Possible 
     * returns for this function are BigDecimal, Double, BigInteger, Long, and Integer.
     * When a Double is returned, it should always be a valid Double and not NaN or +-infinity.
     * 
     * @param val value to convert
     * @return Number representation of the value.
     * @throws NumberFormatException thrown if the value is not a valid number. A public
     *      caller should catch this and wrap it in a {@link JSONException} if applicable.
     */
    protected static Number stringToNumber(final String val) throws NumberFormatException {
        char initial = val.charAt(0);
        if ((initial >= '0' && initial <= '9') || initial == '-') {
            // decimal representation
            if (isDecimalNotation(val)) {
                // quick dirty way to see if we need a BigDecimal instead of a Double
                // this only handles some cases of overflow or underflow
                if (val.length()>14) {
                    return new BigDecimal(val);
                }
                final Double d = Double.valueOf(val);
                if (d.isInfinite() || d.isNaN()) {
                    // if we can't parse it as a double, go up to BigDecimal
                    // this is probably due to underflow like 4.32e-678
                    // or overflow like 4.65e5324. The size of the string is small
                    // but can't be held in a Double.
                    return new BigDecimal(val);
                }
                return d;
            }
            // integer representation.
            // This will narrow any values to the smallest reasonable Object representation
            // (Integer, Long, or BigInteger)
            
            // string version
            // The compare string length method reduces GC,
            // but leads to smaller integers being placed in larger wrappers even though not
            // needed. i.e. 1,000,000,000 -> Long even though it's an Integer
            // 1,000,000,000,000,000,000 -> BigInteger even though it's a Long
            //if(val.length()<=9){
            //    return Integer.valueOf(val);
            //}
            //if(val.length()<=18){
            //    return Long.valueOf(val);
            //}
            //return new BigInteger(val);
            
            // BigInteger version: We use a similar bitLength compare as
            // BigInteger#intValueExact uses. Increases GC, but objects hold
            // only what they need. i.e. Less runtime overhead if the value is
            // long lived. Which is the better tradeoff? This is closer to what's
            // in stringToValue.
            BigInteger bi = new BigInteger(val);
            if(bi.bitLength()<=31){
                return Integer.valueOf(bi.intValue());
            }
            if(bi.bitLength()<=63){
                return Long.valueOf(bi.longValue());
            }
            return bi;
        }
        throw new NumberFormatException("val ["+val+"] is not a valid number.");
    }

    /**
     * Try to convert a string into a number, boolean, or null. If the string
     * can't be converted, return the string.
     *
     * @param string
     *            A String. can not be null.
     * @return A simple JSON value.
     * @throws NullPointerException
     *             Thrown if the string is null.
     */
    // Changes to this method must be copied to the corresponding method in
    // the XML class to keep full support for Android
    public static Object stringToValue(String string) {
        if ("".equals(string)) {
            return string;
        }

        // check JSON key words true/false/null
        if ("true".equalsIgnoreCase(string)) {
            return Boolean.TRUE;
        }
        if ("false".equalsIgnoreCase(string)) {
            return Boolean.FALSE;
        }
        if ("null".equalsIgnoreCase(string)) {
            return JSONObject.NULL;
        }

        /*
         * If it might be a number, try converting it. If a number cannot be
         * produced, then the value will just be a string.
         */

        char initial = string.charAt(0);
        if ((initial >= '0' && initial <= '9') || initial == '-') {
            try {
                // if we want full Big Number support the contents of this
                // `try` block can be replaced with:
                // return stringToNumber(string);
                if (isDecimalNotation(string)) {
                    Double d = Double.valueOf(string);
                    if (!d.isInfinite() && !d.isNaN()) {
                        return d;
                    }
                } else {
                    Long myLong = Long.valueOf(string);
                    if (string.equals(myLong.toString())) {
                        if (myLong.longValue() == myLong.intValue()) {
                            return Integer.valueOf(myLong.intValue());
                        }
                        return myLong;
                    }
                }
            } catch (Exception ignore) {
            }
        }
        return string;
    }

    /**
     * Throw an exception if the object is a NaN or infinite number.
     *
     * @param o
     *            The object to test.
     * @throws JSONException
     *             If o is a non-finite number.
     */
    public static void testValidity(Object o) throws JSONException {
        if (o != null) {
            if (o instanceof Double) {
                if (((Double) o).isInfinite() || ((Double) o).isNaN()) {
                    throw new JSONException(
                            "JSON does not allow non-finite numbers.");
                }
            } else if (o instanceof Float) {
                if (((Float) o).isInfinite() || ((Float) o).isNaN()) {
                    throw new JSONException(
                            "JSON does not allow non-finite numbers.");
                }
            }
        }
    }

    /**
     * Produce a JSONArray containing the values of the members of this
     * JSONObject.
     *
     * @param names
     *            A JSONArray containing a list of key strings. This determines
     *            the sequence of the values in the result.
     * @return A JSONArray of values.
     * @throws JSONException
     *             If any of the values are non-finite numbers.
     */
    public JSONArray toJSONArray(JSONArray names) throws JSONException {
        if (names == null || names.isEmpty()) {
            return null;
        }
        JSONArray ja = new JSONArray();
        for (int i = 0; i < names.length(); i += 1) {
            ja.put(this.opt(names.getString(i)));
        }
        return ja;
    }

    /**
     * Make a JSON text of this JSONObject. For compactness, no whitespace is
     * added. If this would not result in a syntactically correct JSON text,
     * then null will be returned instead.
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     * 
     * @return a printable, displayable, portable, transmittable representation
     *         of the object, beginning with <code>{</code>&nbsp;<small>(left
     *         brace)</small> and ending with <code>}</code>&nbsp;<small>(right
     *         brace)</small>.
     */
    @Override
    public String toString() {
        try {
            return this.toString(0);
        } catch (Exception e) {
            return null;
        }
    }

    /**
     * Make a pretty-printed JSON text of this JSONObject.
     * 
     * <p>If <code>indentFactor > 0</code> and the {@link JSONObject}
     * has only one key, then the object will be output on a single line:
     * <pre>{@code {"key": 1}}</pre>
     * 
     * <p>If an object has 2 or more keys, then it will be output across
     * multiple lines: <code><pre>{
     *  "key1": 1,
     *  "key2": "value 2",
     *  "key3": 3
     * }</pre></code>
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     *
     * @param indentFactor
     *            The number of spaces to add to each level of indentation.
     * @return a printable, displayable, portable, transmittable representation
     *         of the object, beginning with <code>{</code>&nbsp;<small>(left
     *         brace)</small> and ending with <code>}</code>&nbsp;<small>(right
     *         brace)</small>.
     * @throws JSONException
     *             If the object contains an invalid number.
     */
    public String toString(int indentFactor) throws JSONException {
        StringWriter w = new StringWriter();
        synchronized (w.getBuffer()) {
            return this.write(w, indentFactor, 0).toString();
        }
    }

    /**
     * Make a JSON text of an Object value. If the object has an
     * value.toJSONString() method, then that method will be used to produce the
     * JSON text. The method is required to produce a strictly conforming text.
     * If the object does not contain a toJSONString method (which is the most
     * common case), then a text will be produced by other means. If the value
     * is an array or Collection, then a JSONArray will be made from it and its
     * toJSONString method will be called. If the value is a MAP, then a
     * JSONObject will be made from it and its toJSONString method will be
     * called. Otherwise, the value's toString method will be called, and the
     * result will be quoted.
     *
     * <p>
     * Warning: This method assumes that the data structure is acyclical.
     *
     * @param value
     *            The value to be serialized.
     * @return a printable, displayable, transmittable representation of the
     *         object, beginning with <code>{</code>&nbsp;<small>(left
     *         brace)</small> and ending with <code>}</code>&nbsp;<small>(right
     *         brace)</small>.
     * @throws JSONException
     *             If the value is or contains an invalid number.
     */
    public static String valueToString(Object value) throws JSONException {
    	// moves the implementation to JSONWriter as:
    	// 1. It makes more sense to be part of the writer class
    	// 2. For Android support this method is not available. By implementing it in the Writer
    	//    Android users can use the writer with the built in Android JSONObject implementation.
        return JSONWriter.valueToString(value);
    }

    /**
     * Wrap an object, if necessary. If the object is <code>null</code>, return the NULL
     * object. If it is an array or collection, wrap it in a JSONArray. If it is
     * a map, wrap it in a JSONObject. If it is a standard property (Double,
     * String, et al) then it is already wrapped. Otherwise, if it comes from
     * one of the java packages, turn it into a string. And if it doesn't, try
     * to wrap it in a JSONObject. If the wrapping fails, then null is returned.
     *
     * @param object
     *            The object to wrap
     * @return The wrapped value
     */
    public static Object wrap(Object object) {
        try {
            if (object == null) {
                return NULL;
            }
            if (object instanceof JSONObject || object instanceof JSONArray
                    || NULL.equals(object) || object instanceof JSONString
                    || object instanceof Byte || object instanceof Character
                    || object instanceof Short || object instanceof Integer
                    || object instanceof Long || object instanceof Boolean
                    || object instanceof Float || object instanceof Double
                    || object instanceof String || object instanceof BigInteger
                    || object instanceof BigDecimal || object instanceof Enum) {
                return object;
            }

            if (object instanceof Collection) {
                Collection<?> coll = (Collection<?>) object;
                return new JSONArray(coll);
            }
            if (object.getClass().isArray()) {
                return new JSONArray(object);
            }
            if (object instanceof Map) {
                Map<?, ?> map = (Map<?, ?>) object;
                return new JSONObject(map);
            }
            Package objectPackage = object.getClass().getPackage();
            String objectPackageName = objectPackage != null ? objectPackage
                    .getName() : "";
            if (objectPackageName.startsWith("java.")
                    || objectPackageName.startsWith("javax.")
                    || object.getClass().getClassLoader() == null) {
                return object.toString();
            }
            return new JSONObject(object);
        } catch (Exception exception) {
            return null;
        }
    }

    /**
     * Write the contents of the JSONObject as JSON text to a writer. For
     * compactness, no whitespace is added.
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     * 
     * @return The writer.
     * @throws JSONException
     */
    public Writer write(Writer writer) throws JSONException {
        return this.write(writer, 0, 0);
    }

    static final Writer writeValue(Writer writer, Object value,
            int indentFactor, int indent) throws JSONException, IOException {
        if (value == null || value.equals(null)) {
            writer.write("null");
        } else if (value instanceof JSONString) {
            Object o;
            try {
                o = ((JSONString) value).toJSONString();
            } catch (Exception e) {
                throw new JSONException(e);
            }
            writer.write(o != null ? o.toString() : quote(value.toString()));
        } else if (value instanceof Number) {
            // not all Numbers may match actual JSON Numbers. i.e. fractions or Imaginary
            final String numberAsString = numberToString((Number) value);
            if(NUMBER_PATTERN.matcher(numberAsString).matches()) {
                writer.write(numberAsString);
            } else {
                // The Number value is not a valid JSON number.
                // Instead we will quote it as a string
                quote(numberAsString, writer);
            }
        } else if (value instanceof Boolean) {
            writer.write(value.toString());
        } else if (value instanceof Enum<?>) {
            writer.write(quote(((Enum<?>)value).name()));
        } else if (value instanceof JSONObject) {
            ((JSONObject) value).write(writer, indentFactor, indent);
        } else if (value instanceof JSONArray) {
            ((JSONArray) value).write(writer, indentFactor, indent);
        } else if (value instanceof Map) {
            Map<?, ?> map = (Map<?, ?>) value;
            new JSONObject(map).write(writer, indentFactor, indent);
        } else if (value instanceof Collection) {
            Collection<?> coll = (Collection<?>) value;
            new JSONArray(coll).write(writer, indentFactor, indent);
        } else if (value.getClass().isArray()) {
            new JSONArray(value).write(writer, indentFactor, indent);
        } else {
            quote(value.toString(), writer);
        }
        return writer;
    }

    static final void indent(Writer writer, int indent) throws IOException {
        for (int i = 0; i < indent; i += 1) {
            writer.write(' ');
        }
    }

    /**
     * Write the contents of the JSONObject as JSON text to a writer.
     * 
     * <p>If <code>indentFactor > 0</code> and the {@link JSONObject}
     * has only one key, then the object will be output on a single line:
     * <pre>{@code {"key": 1}}</pre>
     * 
     * <p>If an object has 2 or more keys, then it will be output across
     * multiple lines: <code><pre>{
     *  "key1": 1,
     *  "key2": "value 2",
     *  "key3": 3
     * }</pre></code>
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     *
     * @param writer
     *            Writes the serialized JSON
     * @param indentFactor
     *            The number of spaces to add to each level of indentation.
     * @param indent
     *            The indentation of the top level.
     * @return The writer.
     * @throws JSONException
     */
    public Writer write(Writer writer, int indentFactor, int indent)
            throws JSONException {
        try {
            boolean needsComma = false;
            final int length = this.length();
            writer.write('{');

            if (length == 1) {
            	final Entry<String,?> entry = this.entrySet().iterator().next();
                final String key = entry.getKey();
                writer.write(quote(key));
                writer.write(':');
                if (indentFactor > 0) {
                    writer.write(' ');
                }
                try{
                    writeValue(writer, entry.getValue(), indentFactor, indent);
                } catch (Exception e) {
                    throw new JSONException("Unable to write JSONObject value for key: " + key, e);
                }
            } else if (length != 0) {
                final int newIndent = indent + indentFactor;
                for (final Entry<String,?> entry : this.entrySet()) {
                    if (needsComma) {
                        writer.write(',');
                    }
                    if (indentFactor > 0) {
                        writer.write('\n');
                    }
                    indent(writer, newIndent);
                    final String key = entry.getKey();
                    writer.write(quote(key));
                    writer.write(':');
                    if (indentFactor > 0) {
                        writer.write(' ');
                    }
                    try {
                        writeValue(writer, entry.getValue(), indentFactor, newIndent);
                    } catch (Exception e) {
                        throw new JSONException("Unable to write JSONObject value for key: " + key, e);
                    }
                    needsComma = true;
                }
                if (indentFactor > 0) {
                    writer.write('\n');
                }
                indent(writer, indent);
            }
            writer.write('}');
            return writer;
        } catch (IOException exception) {
            throw new JSONException(exception);
        }
    }

    /**
     * Returns a java.util.Map containing all of the entries in this object.
     * If an entry in the object is a JSONArray or JSONObject it will also
     * be converted.
     * <p>
     * Warning: This method assumes that the data structure is acyclical.
     *
     * @return a java.util.Map containing the entries of this object
     */
    public Map<String, Object> toMap() {
        Map<String, Object> results = new HashMap<String, Object>();
        for (Entry<String, Object> entry : this.entrySet()) {
            Object value;
            if (entry.getValue() == null || NULL.equals(entry.getValue())) {
                value = null;
            } else if (entry.getValue() instanceof JSONObject) {
                value = ((JSONObject) entry.getValue()).toMap();
            } else if (entry.getValue() instanceof JSONArray) {
                value = ((JSONArray) entry.getValue()).toList();
            } else {
                value = entry.getValue();
            }
            results.put(entry.getKey(), value);
        }
        return results;
    }
    
    /**
     * Create a new JSONException in a common format for incorrect conversions.
     * @param key name of the key
     * @param valueType the type of value being coerced to
     * @param cause optional cause of the coercion failure
     * @return JSONException that can be thrown.
     */
    private static JSONException wrongValueFormatException(
            String key,
            String valueType,
            Throwable cause) {
        return new JSONException(
                "JSONObject[" + quote(key) + "] is not a " + valueType + "."
                , cause);
    }
    
    /**
     * Create a new JSONException in a common format for incorrect conversions.
     * @param key name of the key
     * @param valueType the type of value being coerced to
     * @param cause optional cause of the coercion failure
     * @return JSONException that can be thrown.
     */
    private static JSONException wrongValueFormatException(
            String key,
            String valueType,
            Object value,
            Throwable cause) {
        return new JSONException(
                "JSONObject[" + quote(key) + "] is not a " + valueType + " (" + value + ")."
                , cause);
    }
} public class JSONArray implements Iterable<Object> {

    /**
     * The arrayList where the JSONArray's properties are kept.
     */
    private final ArrayList<Object> myArrayList;

    /**
     * Construct an empty JSONArray.
     */
    public JSONArray() {
        this.myArrayList = new ArrayList<Object>();
    }

    /**
     * Construct a JSONArray from a JSONTokener.
     *
     * @param x
     *            A JSONTokener
     * @throws JSONException
     *             If there is a syntax error.
     */
    public JSONArray(JSONTokener x) throws JSONException {
        this();
        if (x.nextClean() != '[') {
            throw x.syntaxError("A JSONArray text must start with '['");
        }
        
        char nextChar = x.nextClean();
        if (nextChar == 0) {
            // array is unclosed. No ']' found, instead EOF
            throw x.syntaxError("Expected a ',' or ']'");
        }
        if (nextChar != ']') {
            x.back();
            for (;;) {
                if (x.nextClean() == ',') {
                    x.back();
                    this.myArrayList.add(JSONObject.NULL);
                } else {
                    x.back();
                    this.myArrayList.add(x.nextValue());
                }
                switch (x.nextClean()) {
                case 0:
                    // array is unclosed. No ']' found, instead EOF
                    throw x.syntaxError("Expected a ',' or ']'");
                case ',':
                    nextChar = x.nextClean();
                    if (nextChar == 0) {
                        // array is unclosed. No ']' found, instead EOF
                        throw x.syntaxError("Expected a ',' or ']'");
                    }
                    if (nextChar == ']') {
                        return;
                    }
                    x.back();
                    break;
                case ']':
                    return;
                default:
                    throw x.syntaxError("Expected a ',' or ']'");
                }
            }
        }
    }

    /**
     * Construct a JSONArray from a source JSON text.
     *
     * @param source
     *            A string that begins with <code>[</code>&nbsp;<small>(left
     *            bracket)</small> and ends with <code>]</code>
     *            &nbsp;<small>(right bracket)</small>.
     * @throws JSONException
     *             If there is a syntax error.
     */
    public JSONArray(String source) throws JSONException {
        this(new JSONTokener(source));
    }

    /**
     * Construct a JSONArray from a Collection.
     *
     * @param collection
     *            A Collection.
     */
    public JSONArray(Collection<?> collection) {
        if (collection == null) {
            this.myArrayList = new ArrayList<Object>();
        } else {
            this.myArrayList = new ArrayList<Object>(collection.size());
        	for (Object o: collection){
        		this.myArrayList.add(JSONObject.wrap(o));
        	}
        }
    }

    /**
     * Construct a JSONArray from an array.
     *
     * @param array
     *            Array. If the parameter passed is null, or not an array, an
     *            exception will be thrown.
     *
     * @throws JSONException
     *            If not an array or if an array value is non-finite number.
     * @throws NullPointerException
     *            Thrown if the array parameter is null.
     */
    public JSONArray(Object array) throws JSONException {
        this();
        if (array.getClass().isArray()) {
            int length = Array.getLength(array);
            this.myArrayList.ensureCapacity(length);
            for (int i = 0; i < length; i += 1) {
                this.put(JSONObject.wrap(Array.get(array, i)));
            }
        } else {
            throw new JSONException(
                    "JSONArray initial value should be a string or collection or array.");
        }
    }

    @Override
    public Iterator<Object> iterator() {
        return this.myArrayList.iterator();
    }

    /**
     * Get the object value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return An object value.
     * @throws JSONException
     *             If there is no value for the index.
     */
    public Object get(int index) throws JSONException {
        Object object = this.opt(index);
        if (object == null) {
            throw new JSONException("JSONArray[" + index + "] not found.");
        }
        return object;
    }

    /**
     * Get the boolean value associated with an index. The string values "true"
     * and "false" are converted to boolean.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The truth.
     * @throws JSONException
     *             If there is no value for the index or if the value is not
     *             convertible to boolean.
     */
    public boolean getBoolean(int index) throws JSONException {
        Object object = this.get(index);
        if (object.equals(Boolean.FALSE)
                || (object instanceof String && ((String) object)
                        .equalsIgnoreCase("false"))) {
            return false;
        } else if (object.equals(Boolean.TRUE)
                || (object instanceof String && ((String) object)
                        .equalsIgnoreCase("true"))) {
            return true;
        }
        throw wrongValueFormatException(index, "boolean", null);
    }

    /**
     * Get the double value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     * @throws JSONException
     *             If the key is not found or if the value cannot be converted
     *             to a number.
     */
    public double getDouble(int index) throws JSONException {
        final Object object = this.get(index);
        if(object instanceof Number) {
            return ((Number)object).doubleValue();
        }
        try {
            return Double.parseDouble(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(index, "double", e);
        }
    }

    /**
     * Get the float value associated with a key.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value is not a Number
     *             object and cannot be converted to a number.
     */
    public float getFloat(int index) throws JSONException {
        final Object object = this.get(index);
        if(object instanceof Number) {
            return ((Float)object).floatValue();
        }
        try {
            return Float.parseFloat(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(index, "float", e);
        }
    }

    /**
     * Get the Number value associated with a key.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The numeric value.
     * @throws JSONException
     *             if the key is not found or if the value is not a Number
     *             object and cannot be converted to a number.
     */
    public Number getNumber(int index) throws JSONException {
        Object object = this.get(index);
        try {
            if (object instanceof Number) {
                return (Number)object;
            }
            return JSONObject.stringToNumber(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(index, "number", e);
        }
    }

    /**
     * Get the enum value associated with an index.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *            The type of enum to retrieve.
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The enum value at the index location
     * @throws JSONException
     *            if the key is not found or if the value cannot be converted
     *            to an enum.
     */
    public <E extends Enum<E>> E getEnum(Class<E> clazz, int index) throws JSONException {
        E val = optEnum(clazz, index);
        if(val==null) {
            // JSONException should really take a throwable argument.
            // If it did, I would re-implement this with the Enum.valueOf
            // method and place any thrown exception in the JSONException
            throw wrongValueFormatException(index, "enum of type "
                    + JSONObject.quote(clazz.getSimpleName()), null);
        }
        return val;
    }

    /**
     * Get the BigDecimal value associated with an index. If the value is float
     * or double, the the {@link BigDecimal#BigDecimal(double)} constructor
     * will be used. See notes on the constructor for conversion issues that
     * may arise.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     * @throws JSONException
     *             If the key is not found or if the value cannot be converted
     *             to a BigDecimal.
     */
    public BigDecimal getBigDecimal (int index) throws JSONException {
        Object object = this.get(index);
        BigDecimal val = JSONObject.objectToBigDecimal(object, null);
        if(val == null) {
            throw wrongValueFormatException(index, "BigDecimal", object, null);
        }
        return val;
    }

    /**
     * Get the BigInteger value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     * @throws JSONException
     *             If the key is not found or if the value cannot be converted
     *             to a BigInteger.
     */
    public BigInteger getBigInteger (int index) throws JSONException {
        Object object = this.get(index);
        BigInteger val = JSONObject.objectToBigInteger(object, null);
        if(val == null) {
            throw wrongValueFormatException(index, "BigInteger", object, null);
        }
        return val;
    }

    /**
     * Get the int value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     * @throws JSONException
     *             If the key is not found or if the value is not a number.
     */
    public int getInt(int index) throws JSONException {
        final Object object = this.get(index);
        if(object instanceof Number) {
            return ((Number)object).intValue();
        }
        try {
            return Integer.parseInt(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(index, "int", e);
        }
    }

    /**
     * Get the JSONArray associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return A JSONArray value.
     * @throws JSONException
     *             If there is no value for the index. or if the value is not a
     *             JSONArray
     */
    public JSONArray getJSONArray(int index) throws JSONException {
        Object object = this.get(index);
        if (object instanceof JSONArray) {
            return (JSONArray) object;
        }
        throw wrongValueFormatException(index, "JSONArray", null);
    }

    /**
     * Get the JSONObject associated with an index.
     *
     * @param index
     *            subscript
     * @return A JSONObject value.
     * @throws JSONException
     *             If there is no value for the index or if the value is not a
     *             JSONObject
     */
    public JSONObject getJSONObject(int index) throws JSONException {
        Object object = this.get(index);
        if (object instanceof JSONObject) {
            return (JSONObject) object;
        }
        throw wrongValueFormatException(index, "JSONObject", null);
    }

    /**
     * Get the long value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     * @throws JSONException
     *             If the key is not found or if the value cannot be converted
     *             to a number.
     */
    public long getLong(int index) throws JSONException {
        final Object object = this.get(index);
        if(object instanceof Number) {
            return ((Number)object).longValue();
        }
        try {
            return Long.parseLong(object.toString());
        } catch (Exception e) {
            throw wrongValueFormatException(index, "long", e);
        }
    }

    /**
     * Get the string associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return A string value.
     * @throws JSONException
     *             If there is no string value for the index.
     */
    public String getString(int index) throws JSONException {
        Object object = this.get(index);
        if (object instanceof String) {
            return (String) object;
        }
        throw wrongValueFormatException(index, "String", null);
    }

    /**
     * Determine if the value is <code>null</code>.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return true if the value at the index is <code>null</code>, or if there is no value.
     */
    public boolean isNull(int index) {
        return JSONObject.NULL.equals(this.opt(index));
    }

    /**
     * Make a string from the contents of this JSONArray. The
     * <code>separator</code> string is inserted between each element. Warning:
     * This method assumes that the data structure is acyclical.
     *
     * @param separator
     *            A string that will be inserted between the elements.
     * @return a string.
     * @throws JSONException
     *             If the array contains an invalid number.
     */
    public String join(String separator) throws JSONException {
        int len = this.length();
        if (len == 0) {
            return "";
        }
        
        StringBuilder sb = new StringBuilder(
                   JSONObject.valueToString(this.myArrayList.get(0)));

        for (int i = 1; i < len; i++) {
            sb.append(separator)
              .append(JSONObject.valueToString(this.myArrayList.get(i)));
        }
        return sb.toString();
    }

    /**
     * Get the number of elements in the JSONArray, included nulls.
     *
     * @return The length (or size).
     */
    public int length() {
        return this.myArrayList.size();
    }

    /**
     * Get the optional object value associated with an index.
     *
     * @param index
     *            The index must be between 0 and length() - 1. If not, null is returned.
     * @return An object value, or null if there is no object at that index.
     */
    public Object opt(int index) {
        return (index < 0 || index >= this.length()) ? null : this.myArrayList
                .get(index);
    }

    /**
     * Get the optional boolean value associated with an index. It returns false
     * if there is no value at that index, or if the value is not Boolean.TRUE
     * or the String "true".
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The truth.
     */
    public boolean optBoolean(int index) {
        return this.optBoolean(index, false);
    }

    /**
     * Get the optional boolean value associated with an index. It returns the
     * defaultValue if there is no value at that index or if it is not a Boolean
     * or the String "true" or "false" (case insensitive).
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            A boolean default.
     * @return The truth.
     */
    public boolean optBoolean(int index, boolean defaultValue) {
        try {
            return this.getBoolean(index);
        } catch (Exception e) {
            return defaultValue;
        }
    }

    /**
     * Get the optional double value associated with an index. NaN is returned
     * if there is no value for the index, or if the value is not a number and
     * cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     */
    public double optDouble(int index) {
        return this.optDouble(index, Double.NaN);
    }

    /**
     * Get the optional double value associated with an index. The defaultValue
     * is returned if there is no value for the index, or if the value is not a
     * number and cannot be converted to a number.
     *
     * @param index
     *            subscript
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public double optDouble(int index, double defaultValue) {
        final Number val = this.optNumber(index, null);
        if (val == null) {
            return defaultValue;
        }
        final double doubleValue = val.doubleValue();
        // if (Double.isNaN(doubleValue) || Double.isInfinite(doubleValue)) {
        // return defaultValue;
        // }
        return doubleValue;
    }

    /**
     * Get the optional float value associated with an index. NaN is returned
     * if there is no value for the index, or if the value is not a number and
     * cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     */
    public float optFloat(int index) {
        return this.optFloat(index, Float.NaN);
    }

    /**
     * Get the optional float value associated with an index. The defaultValue
     * is returned if there is no value for the index, or if the value is not a
     * number and cannot be converted to a number.
     *
     * @param index
     *            subscript
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public float optFloat(int index, float defaultValue) {
        final Number val = this.optNumber(index, null);
        if (val == null) {
            return defaultValue;
        }
        final float floatValue = val.floatValue();
        // if (Float.isNaN(floatValue) || Float.isInfinite(floatValue)) {
        // return floatValue;
        // }
        return floatValue;
    }

    /**
     * Get the optional int value associated with an index. Zero is returned if
     * there is no value for the index, or if the value is not a number and
     * cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     */
    public int optInt(int index) {
        return this.optInt(index, 0);
    }

    /**
     * Get the optional int value associated with an index. The defaultValue is
     * returned if there is no value for the index, or if the value is not a
     * number and cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public int optInt(int index, int defaultValue) {
        final Number val = this.optNumber(index, null);
        if (val == null) {
            return defaultValue;
        }
        return val.intValue();
    }

    /**
     * Get the enum value associated with a key.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *            The type of enum to retrieve.
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The enum value at the index location or null if not found
     */
    public <E extends Enum<E>> E optEnum(Class<E> clazz, int index) {
        return this.optEnum(clazz, index, null);
    }

    /**
     * Get the enum value associated with a key.
     * 
     * @param <E>
     *            Enum Type
     * @param clazz
     *            The type of enum to retrieve.
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default in case the value is not found
     * @return The enum value at the index location or defaultValue if
     *            the value is not found or cannot be assigned to clazz
     */
    public <E extends Enum<E>> E optEnum(Class<E> clazz, int index, E defaultValue) {
        try {
            Object val = this.opt(index);
            if (JSONObject.NULL.equals(val)) {
                return defaultValue;
            }
            if (clazz.isAssignableFrom(val.getClass())) {
                // we just checked it!
                @SuppressWarnings("unchecked")
                E myE = (E) val;
                return myE;
            }
            return Enum.valueOf(clazz, val.toString());
        } catch (IllegalArgumentException e) {
            return defaultValue;
        } catch (NullPointerException e) {
            return defaultValue;
        }
    }

    /**
     * Get the optional BigInteger value associated with an index. The 
     * defaultValue is returned if there is no value for the index, or if the 
     * value is not a number and cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public BigInteger optBigInteger(int index, BigInteger defaultValue) {
        Object val = this.opt(index);
        return JSONObject.objectToBigInteger(val, defaultValue);
    }

    /**
     * Get the optional BigDecimal value associated with an index. The 
     * defaultValue is returned if there is no value for the index, or if the 
     * value is not a number and cannot be converted to a number. If the value
     * is float or double, the the {@link BigDecimal#BigDecimal(double)}
     * constructor will be used. See notes on the constructor for conversion
     * issues that may arise.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public BigDecimal optBigDecimal(int index, BigDecimal defaultValue) {
        Object val = this.opt(index);
        return JSONObject.objectToBigDecimal(val, defaultValue);
    }

    /**
     * Get the optional JSONArray associated with an index.
     *
     * @param index
     *            subscript
     * @return A JSONArray value, or null if the index has no value, or if the
     *         value is not a JSONArray.
     */
    public JSONArray optJSONArray(int index) {
        Object o = this.opt(index);
        return o instanceof JSONArray ? (JSONArray) o : null;
    }

    /**
     * Get the optional JSONObject associated with an index. Null is returned if
     * the key is not found, or null if the index has no value, or if the value
     * is not a JSONObject.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return A JSONObject value.
     */
    public JSONObject optJSONObject(int index) {
        Object o = this.opt(index);
        return o instanceof JSONObject ? (JSONObject) o : null;
    }

    /**
     * Get the optional long value associated with an index. Zero is returned if
     * there is no value for the index, or if the value is not a number and
     * cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return The value.
     */
    public long optLong(int index) {
        return this.optLong(index, 0);
    }

    /**
     * Get the optional long value associated with an index. The defaultValue is
     * returned if there is no value for the index, or if the value is not a
     * number and cannot be converted to a number.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default value.
     * @return The value.
     */
    public long optLong(int index, long defaultValue) {
        final Number val = this.optNumber(index, null);
        if (val == null) {
            return defaultValue;
        }
        return val.longValue();
    }

    /**
     * Get an optional {@link Number} value associated with a key, or <code>null</code>
     * if there is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number ({@link BigDecimal}). This method
     * would be used in cases where type coercion of the number value is unwanted.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return An object which is the value.
     */
    public Number optNumber(int index) {
        return this.optNumber(index, null);
    }

    /**
     * Get an optional {@link Number} value associated with a key, or the default if there
     * is no such key or if the value is not a number. If the value is a string,
     * an attempt will be made to evaluate it as a number ({@link BigDecimal}). This method
     * would be used in cases where type coercion of the number value is unwanted.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default.
     * @return An object which is the value.
     */
    public Number optNumber(int index, Number defaultValue) {
        Object val = this.opt(index);
        if (JSONObject.NULL.equals(val)) {
            return defaultValue;
        }
        if (val instanceof Number){
            return (Number) val;
        }
        
        if (val instanceof String) {
            try {
                return JSONObject.stringToNumber((String) val);
            } catch (Exception e) {
                return defaultValue;
            }
        }
        return defaultValue;
    }

    /**
     * Get the optional string value associated with an index. It returns an
     * empty string if there is no value at that index. If the value is not a
     * string and is not null, then it is converted to a string.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @return A String value.
     */
    public String optString(int index) {
        return this.optString(index, "");
    }

    /**
     * Get the optional string associated with an index. The defaultValue is
     * returned if the key is not found.
     *
     * @param index
     *            The index must be between 0 and length() - 1.
     * @param defaultValue
     *            The default value.
     * @return A String value.
     */
    public String optString(int index, String defaultValue) {
        Object object = this.opt(index);
        return JSONObject.NULL.equals(object) ? defaultValue : object
                .toString();
    }

    /**
     * Append a boolean value. This increases the array's length by one.
     *
     * @param value
     *            A boolean value.
     * @return this.
     */
    public JSONArray put(boolean value) {
        return this.put(value ? Boolean.TRUE : Boolean.FALSE);
    }

    /**
     * Put a value in the JSONArray, where the value will be a JSONArray which
     * is produced from a Collection.
     *
     * @param value
     *            A Collection value.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     */
    public JSONArray put(Collection<?> value) {
        return this.put(new JSONArray(value));
    }

    /**
     * Append a double value. This increases the array's length by one.
     *
     * @param value
     *            A double value.
     * @return this.
     * @throws JSONException
     *             if the value is not finite.
     */
    public JSONArray put(double value) throws JSONException {
        return this.put(Double.valueOf(value));
    }
    
    /**
     * Append a float value. This increases the array's length by one.
     *
     * @param value
     *            A float value.
     * @return this.
     * @throws JSONException
     *             if the value is not finite.
     */
    public JSONArray put(float value) throws JSONException {
        return this.put(Float.valueOf(value));
    }

    /**
     * Append an int value. This increases the array's length by one.
     *
     * @param value
     *            An int value.
     * @return this.
     */
    public JSONArray put(int value) {
        return this.put(Integer.valueOf(value));
    }

    /**
     * Append an long value. This increases the array's length by one.
     *
     * @param value
     *            A long value.
     * @return this.
     */
    public JSONArray put(long value) {
        return this.put(Long.valueOf(value));
    }

    /**
     * Put a value in the JSONArray, where the value will be a JSONObject which
     * is produced from a Map.
     *
     * @param value
     *            A Map value.
     * @return this.
     * @throws JSONException
     *            If a value in the map is non-finite number.
     * @throws NullPointerException
     *            If a key in the map is <code>null</code>
     */
    public JSONArray put(Map<?, ?> value) {
        return this.put(new JSONObject(value));
    }

    /**
     * Append an object value. This increases the array's length by one.
     *
     * @param value
     *            An object value. The value should be a Boolean, Double,
     *            Integer, JSONArray, JSONObject, Long, or String, or the
     *            JSONObject.NULL object.
     * @return this.
     * @throws JSONException
     *            If the value is non-finite number.
     */
    public JSONArray put(Object value) {
        JSONObject.testValidity(value);
        this.myArrayList.add(value);
        return this;
    }

    /**
     * Put or replace a boolean value in the JSONArray. If the index is greater
     * than the length of the JSONArray, then null elements will be added as
     * necessary to pad it out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            A boolean value.
     * @return this.
     * @throws JSONException
     *             If the index is negative.
     */
    public JSONArray put(int index, boolean value) throws JSONException {
        return this.put(index, value ? Boolean.TRUE : Boolean.FALSE);
    }

    /**
     * Put a value in the JSONArray, where the value will be a JSONArray which
     * is produced from a Collection.
     *
     * @param index
     *            The subscript.
     * @param value
     *            A Collection value.
     * @return this.
     * @throws JSONException
     *             If the index is negative or if the value is non-finite.
     */
    public JSONArray put(int index, Collection<?> value) throws JSONException {
        return this.put(index, new JSONArray(value));
    }

    /**
     * Put or replace a double value. If the index is greater than the length of
     * the JSONArray, then null elements will be added as necessary to pad it
     * out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            A double value.
     * @return this.
     * @throws JSONException
     *             If the index is negative or if the value is non-finite.
     */
    public JSONArray put(int index, double value) throws JSONException {
        return this.put(index, Double.valueOf(value));
    }

    /**
     * Put or replace a float value. If the index is greater than the length of
     * the JSONArray, then null elements will be added as necessary to pad it
     * out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            A float value.
     * @return this.
     * @throws JSONException
     *             If the index is negative or if the value is non-finite.
     */
    public JSONArray put(int index, float value) throws JSONException {
        return this.put(index, Float.valueOf(value));
    }

    /**
     * Put or replace an int value. If the index is greater than the length of
     * the JSONArray, then null elements will be added as necessary to pad it
     * out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            An int value.
     * @return this.
     * @throws JSONException
     *             If the index is negative.
     */
    public JSONArray put(int index, int value) throws JSONException {
        return this.put(index, Integer.valueOf(value));
    }

    /**
     * Put or replace a long value. If the index is greater than the length of
     * the JSONArray, then null elements will be added as necessary to pad it
     * out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            A long value.
     * @return this.
     * @throws JSONException
     *             If the index is negative.
     */
    public JSONArray put(int index, long value) throws JSONException {
        return this.put(index, Long.valueOf(value));
    }

    /**
     * Put a value in the JSONArray, where the value will be a JSONObject that
     * is produced from a Map.
     *
     * @param index
     *            The subscript.
     * @param value
     *            The Map value.
     * @return this.
     * @throws JSONException
     *             If the index is negative or if the the value is an invalid
     *             number.
     * @throws NullPointerException
     *             If a key in the map is <code>null</code>
     */
    public JSONArray put(int index, Map<?, ?> value) throws JSONException {
        this.put(index, new JSONObject(value));
        return this;
    }

    /**
     * Put or replace an object value in the JSONArray. If the index is greater
     * than the length of the JSONArray, then null elements will be added as
     * necessary to pad it out.
     *
     * @param index
     *            The subscript.
     * @param value
     *            The value to put into the array. The value should be a
     *            Boolean, Double, Integer, JSONArray, JSONObject, Long, or
     *            String, or the JSONObject.NULL object.
     * @return this.
     * @throws JSONException
     *             If the index is negative or if the the value is an invalid
     *             number.
     */
    public JSONArray put(int index, Object value) throws JSONException {
        if (index < 0) {
            throw new JSONException("JSONArray[" + index + "] not found.");
        }
        if (index < this.length()) {
            JSONObject.testValidity(value);
            this.myArrayList.set(index, value);
            return this;
        }
        if(index == this.length()){
            // simple append
            return this.put(value);
        }
        // if we are inserting past the length, we want to grow the array all at once
        // instead of incrementally.
        this.myArrayList.ensureCapacity(index + 1);
        while (index != this.length()) {
            // we don't need to test validity of NULL objects
            this.myArrayList.add(JSONObject.NULL);
        }
        return this.put(value);
    }
    
    /**
     * Creates a JSONPointer using an initialization string and tries to 
     * match it to an item within this JSONArray. For example, given a
     * JSONArray initialized with this document:
     * <pre>
     * [
     *     {"b":"c"}
     * ]
     * </pre>
     * and this JSONPointer string: 
     * <pre>
     * "/0/b"
     * </pre>
     * Then this method will return the String "c"
     * A JSONPointerException may be thrown from code called by this method.
     *
     * @param jsonPointer string that can be used to create a JSONPointer
     * @return the item matched by the JSONPointer, otherwise null
     */
    public Object query(String jsonPointer) {
        return query(new JSONPointer(jsonPointer));
    }
    
    /**
     * Uses a user initialized JSONPointer  and tries to 
     * match it to an item within this JSONArray. For example, given a
     * JSONArray initialized with this document:
     * <pre>
     * [
     *     {"b":"c"}
     * ]
     * </pre>
     * and this JSONPointer: 
     * <pre>
     * "/0/b"
     * </pre>
     * Then this method will return the String "c"
     * A JSONPointerException may be thrown from code called by this method.
     *
     * @param jsonPointer string that can be used to create a JSONPointer
     * @return the item matched by the JSONPointer, otherwise null
     */
    public Object query(JSONPointer jsonPointer) {
        return jsonPointer.queryFrom(this);
    }
    
    /**
     * Queries and returns a value from this object using {@code jsonPointer}, or
     * returns null if the query fails due to a missing key.
     * 
     * @param jsonPointer the string representation of the JSON pointer
     * @return the queried value or {@code null}
     * @throws IllegalArgumentException if {@code jsonPointer} has invalid syntax
     */
    public Object optQuery(String jsonPointer) {
    	return optQuery(new JSONPointer(jsonPointer));
    }
    
    /**
     * Queries and returns a value from this object using {@code jsonPointer}, or
     * returns null if the query fails due to a missing key.
     * 
     * @param jsonPointer The JSON pointer
     * @return the queried value or {@code null}
     * @throws IllegalArgumentException if {@code jsonPointer} has invalid syntax
     */
    public Object optQuery(JSONPointer jsonPointer) {
        try {
            return jsonPointer.queryFrom(this);
        } catch (JSONPointerException e) {
            return null;
        }
    }

    /**
     * Remove an index and close the hole.
     *
     * @param index
     *            The index of the element to be removed.
     * @return The value that was associated with the index, or null if there
     *         was no value.
     */
    public Object remove(int index) {
        return index >= 0 && index < this.length()
            ? this.myArrayList.remove(index)
            : null;
    }

    /**
     * Determine if two JSONArrays are similar.
     * They must contain similar sequences.
     *
     * @param other The other JSONArray
     * @return true if they are equal
     */
    public boolean similar(Object other) {
        if (!(other instanceof JSONArray)) {
            return false;
        }
        int len = this.length();
        if (len != ((JSONArray)other).length()) {
            return false;
        }
        for (int i = 0; i < len; i += 1) {
            Object valueThis = this.myArrayList.get(i);
            Object valueOther = ((JSONArray)other).myArrayList.get(i);
            if(valueThis == valueOther) {
            	continue;
            }
            if(valueThis == null) {
            	return false;
            }
            if (valueThis instanceof JSONObject) {
                if (!((JSONObject)valueThis).similar(valueOther)) {
                    return false;
                }
            } else if (valueThis instanceof JSONArray) {
                if (!((JSONArray)valueThis).similar(valueOther)) {
                    return false;
                }
            } else if (!valueThis.equals(valueOther)) {
                return false;
            }
        }
        return true;
    }

    /**
     * Produce a JSONObject by combining a JSONArray of names with the values of
     * this JSONArray.
     *
     * @param names
     *            A JSONArray containing a list of key strings. These will be
     *            paired with the values.
     * @return A JSONObject, or null if there are no names or if this JSONArray
     *         has no values.
     * @throws JSONException
     *             If any of the names are null.
     */
    public JSONObject toJSONObject(JSONArray names) throws JSONException {
        if (names == null || names.isEmpty() || this.isEmpty()) {
            return null;
        }
        JSONObject jo = new JSONObject(names.length());
        for (int i = 0; i < names.length(); i += 1) {
            jo.put(names.getString(i), this.opt(i));
        }
        return jo;
    }

    /**
     * Make a JSON text of this JSONArray. For compactness, no unnecessary
     * whitespace is added. If it is not possible to produce a syntactically
     * correct JSON text then null will be returned instead. This could occur if
     * the array contains an invalid number.
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     *
     * @return a printable, displayable, transmittable representation of the
     *         array.
     */
    @Override
    public String toString() {
        try {
            return this.toString(0);
        } catch (Exception e) {
            return null;
        }
    }

    /**
     * Make a pretty-printed JSON text of this JSONArray.
     * 
     * <p>If <code>indentFactor > 0</code> and the {@link JSONArray} has only
     * one element, then the array will be output on a single line:
     * <pre>{@code [1]}</pre>
     * 
     * <p>If an array has 2 or more elements, then it will be output across
     * multiple lines: <pre>{@code
     * [
     * 1,
     * "value 2",
     * 3
     * ]
     * }</pre>
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     * 
     * @param indentFactor
     *            The number of spaces to add to each level of indentation.
     * @return a printable, displayable, transmittable representation of the
     *         object, beginning with <code>[</code>&nbsp;<small>(left
     *         bracket)</small> and ending with <code>]</code>
     *         &nbsp;<small>(right bracket)</small>.
     * @throws JSONException
     */
    public String toString(int indentFactor) throws JSONException {
        StringWriter sw = new StringWriter();
        synchronized (sw.getBuffer()) {
            return this.write(sw, indentFactor, 0).toString();
        }
    }

    /**
     * Write the contents of the JSONArray as JSON text to a writer. For
     * compactness, no whitespace is added.
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     *</b>
     *
     * @return The writer.
     * @throws JSONException
     */
    public Writer write(Writer writer) throws JSONException {
        return this.write(writer, 0, 0);
    }

    /**
     * Write the contents of the JSONArray as JSON text to a writer.
     * 
     * <p>If <code>indentFactor > 0</code> and the {@link JSONArray} has only
     * one element, then the array will be output on a single line:
     * <pre>{@code [1]}</pre>
     * 
     * <p>If an array has 2 or more elements, then it will be output across
     * multiple lines: <pre>{@code
     * [
     * 1,
     * "value 2",
     * 3
     * ]
     * }</pre>
     * <p><b>
     * Warning: This method assumes that the data structure is acyclical.
     * </b>
     *
     * @param writer
     *            Writes the serialized JSON
     * @param indentFactor
     *            The number of spaces to add to each level of indentation.
     * @param indent
     *            The indentation of the top level.
     * @return The writer.
     * @throws JSONException
     */
    public Writer write(Writer writer, int indentFactor, int indent)
            throws JSONException {
        try {
            boolean needsComma = false;
            int length = this.length();
            writer.write('[');

            if (length == 1) {
                try {
                    JSONObject.writeValue(writer, this.myArrayList.get(0),
                            indentFactor, indent);
                } catch (Exception e) {
                    throw new JSONException("Unable to write JSONArray value at index: 0", e);
                }
            } else if (length != 0) {
                final int newIndent = indent + indentFactor;

                for (int i = 0; i < length; i += 1) {
                    if (needsComma) {
                        writer.write(',');
                    }
                    if (indentFactor > 0) {
                        writer.write('\n');
                    }
                    JSONObject.indent(writer, newIndent);
                    try {
                        JSONObject.writeValue(writer, this.myArrayList.get(i),
                                indentFactor, newIndent);
                    } catch (Exception e) {
                        throw new JSONException("Unable to write JSONArray value at index: " + i, e);
                    }
                    needsComma = true;
                }
                if (indentFactor > 0) {
                    writer.write('\n');
                }
                JSONObject.indent(writer, indent);
            }
            writer.write(']');
            return writer;
        } catch (IOException e) {
            throw new JSONException(e);
        }
    }

    /**
     * Returns a java.util.List containing all of the elements in this array.
     * If an element in the array is a JSONArray or JSONObject it will also
     * be converted to a List and a Map respectively.
     * <p>
     * Warning: This method assumes that the data structure is acyclical.
     *
     * @return a java.util.List containing the elements of this array
     */
    public List<Object> toList() {
        List<Object> results = new ArrayList<Object>(this.myArrayList.size());
        for (Object element : this.myArrayList) {
            if (element == null || JSONObject.NULL.equals(element)) {
                results.add(null);
            } else if (element instanceof JSONArray) {
                results.add(((JSONArray) element).toList());
            } else if (element instanceof JSONObject) {
                results.add(((JSONObject) element).toMap());
            } else {
                results.add(element);
            }
        }
        return results;
    }

    /**
     * Check if JSONArray is empty.
     *
     * @return true if JSONArray is empty, otherwise false.
     */
    public boolean isEmpty() {
        return this.myArrayList.isEmpty();
    }
    
    /**
     * Create a new JSONException in a common format for incorrect conversions.
     * @param idx index of the item
     * @param valueType the type of value being coerced to
     * @param cause optional cause of the coercion failure
     * @return JSONException that can be thrown.
     */
    private static JSONException wrongValueFormatException(
            int idx,
            String valueType,
            Throwable cause) {
        return new JSONException(
                "JSONArray[" + idx + "] is not a " + valueType + "."
                , cause);
    }
    
    /**
     * Create a new JSONException in a common format for incorrect conversions.
     * @param idx index of the item
     * @param valueType the type of value being coerced to
     * @param cause optional cause of the coercion failure
     * @return JSONException that can be thrown.
     */
    private static JSONException wrongValueFormatException(
            int idx,
            String valueType,
            Object value,
            Throwable cause) {
        return new JSONException(
                "JSONArray[" + idx + "] is not a " + valueType + " (" + value + ")."
                , cause);
    }

} public class Caretaker extends Thread {

    private static final Random random = new Random(System.currentTimeMillis());
    
    private boolean shallRun = true;
    
    public  final static long startupTime = System.currentTimeMillis();
    private final static long upgradeWait = DateParser.DAY_MILLIS; // 1 day
    public        static long upgradeTime = startupTime + upgradeWait;
    private final static long helloPeriod = 600000; // one ping each 10 minutes
    private       static long helloTime   = 0; // latest hello ping time
    private       static long deletionTime = 0;

    private static final int TIMELINE_PUSH_MINSIZE = 200;
    private static final int TIMELINE_PUSH_MAXSIZE = 1000;
    
    /**
     * ask the thread to shut down
     */
    public void shutdown() {
        this.shallRun = false;
        this.interrupt();
        DAO.log("catched caretaker termination signal");
    }
    
    @Override
    public void run() {
        Thread.currentThread().setName("CARETAKER");
        // send a message to other peers that I am alive
        String[] backends = DAO.getBackend();
        int maxRetries = Integer.valueOf(DAO.getConfig("caretaker.backendpush.retries", "5"));
        int backOffParameter = Integer.valueOf(DAO.getConfig("caretaker.backendpush.backoff", "3000"));

        boolean busy = false;
        // work loop
        beat: while (this.shallRun) try {
            // check upgrade time
            if (System.currentTimeMillis() > upgradeTime) {
                // increase the upgrade time to prevent that the peer runs amok (re-tries the attempt all the time) when upgrade fails for any reason
                upgradeTime = upgradeTime + upgradeWait;
                
                // do an upgrade
                DAO.log("UPGRADE: starting an upgrade");
                upgrade();
                DAO.log("UPGRADE: started an upgrade");
            }
            
            // check ping
            if (System.currentTimeMillis() - helloPeriod > helloTime) {
                helloTime = System.currentTimeMillis();
                HelloService.propagate(backends);
            }
            
            // clear caches
            if (SuggestServlet.cache.size() > 100) SuggestServlet.cache.clear();
            
            // sleep a bit to prevent that the DoS limit fires at backend server
            try {Thread.sleep(busy ? 500 : 10000);} catch (InterruptedException e) {}
            if (!this.shallRun) break beat;
            busy = false;
            
            //DAO.log("connection pool: " + ClientConnection.cm.getTotalStats().toString());
            
            // peer-to-peer operation
            TwitterTimeline tl = DAO.outgoingMessages.takeTimelineMin(Order.CREATED_AT, TIMELINE_PUSH_MINSIZE, TIMELINE_PUSH_MAXSIZE);
            int tl_size = tl == null ? 0 : tl.size();
            if (tl_size > 0 && backends.length > 0) {
                // transmit the timeline
                long start = System.currentTimeMillis();
                boolean success = PushServlet.push(backends, tl);
                if (success) {
                    DAO.log("success pushing " + tl.size() + " messages to backend " + Arrays.toString(backends) + " in 1st attempt in " + (System.currentTimeMillis() - start) + " ms");
                } else {
                    // we should try again.. but not an infinite number because then
                    // our timeline in RAM would fill up our RAM creating a memory leak
                    retrylook: for (int retry = 0; retry < maxRetries; retry++) {
                        // give back-end time to recover
                        try {Thread.sleep((retry + 1) * backOffParameter);} catch (InterruptedException e) {}
                        DAO.log("trying to push (again) " + tl.size() + " messages to backend " + Arrays.toString(backends) + ", attempt #" + (retry + 1) + "/5");
                        start = System.currentTimeMillis();
                        if (PushServlet.push(backends, tl)) {
                            DAO.log("success pushing " + tl.size() + " messages to backend " + Arrays.toString(backends) + " in " + (retry + 2) + ". attempt in " + (System.currentTimeMillis() - start) + " ms");
                            success = true;
                            break retrylook;
                        }
                    }
                    if (!success) DAO.log("failed pushing " + tl.size() + " messages to backend " + Arrays.toString(backends));
                }
                busy = true;
            }

            // scan dump input directory to import files
            try {
                DAO.importAccountDumps(Integer.MAX_VALUE);
            } catch (IOException e1) {
                e1.printStackTrace();
            }

            // run some harvesting steps
            boolean retrieval_forbackend_enabled = DAO.getConfig("retrieval.forbackend.enabled", false);
            boolean backend_push_enabled = DAO.getConfig("backend.push.enabled", false);
            int timeline_size = DAO.outgoingMessages.timelineSize();
            if (retrieval_forbackend_enabled && backend_push_enabled && backends.length > 0 && timeline_size < TIMELINE_PUSH_MAXSIZE) {

                int retrieval_forbackend_concurrency = (int) DAO.getConfig("retrieval.forbackend.concurrency", 1);
                int retrieval_forbackend_loops = (int) DAO.getConfig("retrieval.forbackend.loops", 10);
                int retrieval_forbackend_sleep_base = (int) DAO.getConfig("retrieval.forbackend.sleep.base", 300);
                int retrieval_forbackend_sleep_randomoffset = (int) DAO.getConfig("retrieval.forbackend.sleep.randomoffset", 100);
                hloop: for (int i = 0; i < retrieval_forbackend_loops; i++) {

                    // get more queries
                    int pendingQueries = 0;
                    try {
                        pendingQueries = LoklakServer.harvester.get_harvest_queries();
                    } catch (IOException e) {
                        DAO.severe("FAIL SUGGESTIONS cannot get queries from backend: " + e.getMessage(), e);
                        break hloop;
                    }

                    // without queries we cannot do harvesting
                    if (pendingQueries == 0) break hloop;
                    
                    // start harvesting
                    Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)];
                    final AtomicInteger acccount = new AtomicInteger(0);
                    for (int j = 0; j < rts.length; j++) {
                        rts[j] = new Thread() {
                            public void run() {
                                TwitterTimeline tl = LoklakServer.harvester.harvest_timeline();
                                if (tl != null && tl.getQuery() != null) {
                                    /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl);
                                }
                                int count = tl == null ? 0 : tl.size();
                                acccount.addAndGet(count);
                            }
                        };
                        rts[j].start();
                        try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
                    }
                    for (Thread t: rts) t.join();
                    if (acccount.get() < 0) break hloop;
                    try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
                
                    busy = true;
                }
            }

            // run some crawl steps
            if (Crawler.pending() > 0) {
                crawler: for (int i = 0; i < 10; i++) {
                    List<Thread> t = new ArrayList<>();
                    final AtomicBoolean finished = new AtomicBoolean(false);
                    for (int j = 0; j < 10; j++) {
                        Thread u = new Thread() {
                            public void run() {
                                if (Crawler.process() == 0) finished.set(true);
                            }
                        };
                        u.start();
                        t.add(u);
                        try {Thread.sleep(random.nextInt(20));} catch (InterruptedException e) {}
                    }
                    t.forEach(u -> {
                        try {u.join(1000);} catch (InterruptedException e) {}
                    });
                    if (finished.get()) break crawler; else busy = true;
                }
            }

            // run searches
            boolean retrieval_queries_enabled = DAO.getConfig("retrieval.queries.enabled", false);
            if (retrieval_queries_enabled && IncomingMessageBuffer.addSchedulerAvailable()) {
                // execute some queries again: look out in the suggest database for queries with outdated due-time in field retrieval_next
                List<QueryEntry> queryList = DAO.SearchLocalQueries("", 10, "retrieval_next", "date", SortOrder.ASC, null, new Date(), "retrieval_next");
                queriesloop: for (QueryEntry qe: queryList) {
                    if (!acceptQuery4Retrieval(qe.getQuery())) {
                        DAO.deleteQuery(qe.getQuery(), qe.getSourceType());
                        continue queriesloop;
                    }
                    TwitterTimeline t;
                    try {
                        t = DAO.scrapeTwitter(
                                null,
                                qe.getQuery(),
                                Order.CREATED_AT,
                                qe.getTimezoneOffset(),
                                false,
                                10000,
                                true
                        );
                        DAO.announceNewUserId(t);
                    } catch (NullPointerException e) {
                        DAO.severe("TwitterScraper.search() returns null (no twitter results)"
                                + " or any other issue in DAO.scrapeTwitter() method", e);
                        t = new TwitterTimeline(Order.CREATED_AT);
                    }
                    DAO.log("retrieval of " + t.size() + " new messages for q = \"" + qe.getQuery() + "\"");

                    try {
                        Thread.sleep(random.nextInt(200));
                    } catch (InterruptedException e) {}
                    busy = true;
                }
            }
            
            // retrieve user data
            Set<Number> ids = DAO.getNewUserIdsChunk();
            boolean retrieval_user_enabled = DAO.getConfig("retrieval.user.enabled", false);
            if (ids != null && retrieval_user_enabled && TwitterAPI.getAppTwitterFactory() != null) {
                try {
                    TwitterAPI.getScreenName(ids, 10000, false);
                } catch (IOException | TwitterException e) {
                    for (Number n: ids) DAO.announceNewUserId(n); // push back unread values
                    if (e instanceof TwitterException) try {Thread.sleep(10000);} catch (InterruptedException ee) {}
                }
            }
            
            if (!busy) {
                // start a crawler
                String startTerm = DAO.getRandomTerm();
                if (startTerm != null && startTerm.length() > 0) {
                    Crawler.stack(startTerm, 3, true, true, true);
                    DAO.log("started a crawler for term " + startTerm);
                    busy = true;
                }
            }
            
            // heal the latency to give peers with out-dated information a new chance
            DAO.healLatency(0.95f);
            
            // delete messages out of time frames
            // this is IO heavy, so prevent to do this a lot
            if (System.currentTimeMillis() - deletionTime > 60000) {
                int d;
                d = DAO.deleteOld(IndexName.messages_hour, DateParser.oneHourAgo());
                if (d > 0) DAO.log("Deleted " + d + " outdated(hour) messages");
                d = DAO.deleteOld(IndexName.messages_day, DateParser.oneDayAgo());
                if (d > 0) DAO.log("Deleted " + d + " outdated(day) messages");
                d = DAO.deleteOld(IndexName.messages_week, DateParser.oneWeekAgo());
                if (d > 0) DAO.log("Deleted " + d + " outdated(week) messages");
                if (DAO.getConfig("autodeletion", false)) {
                    d = DAO.deleteOld(IndexName.messages, DateParser.oneMonthAgo());
                    if (d > 0) DAO.log("Deleted " + d + " outdated(month) messages");
                }
                deletionTime = System.currentTimeMillis();
            }
        } catch (Throwable e) {
            DAO.severe("CARETAKER THREAD", e);
        }

        DAO.log("caretaker terminated");
    }

    public static boolean acceptQuery4Retrieval(String q) {
        return q.length() > 1 && q.length() <=16 && (q.indexOf(':') < 0 || q.startsWith("from:"));
    }
    
    /**
     * loklak upgrades itself if this is called
     */
    public static void upgrade() {
        final File upgradeScript = new File(DAO.bin_dir.getAbsolutePath().replaceAll(" ", "\\ "), "upgrade.sh");
      
        try {
            List<String> rsp = OS.execSynchronous(upgradeScript.getAbsolutePath());
            for (String s: rsp) DAO.log("UPGRADE: " + s);
        } catch (IOException e) {
        	DAO.severe("UPGRADE failed: " + e.getMessage(), e);
        }
    }
    
} public class SearchServlet extends HttpServlet {

    private static final long serialVersionUID = 563533152152063908L;

    private final static String SEARCH_LOW_COUNT_NAME = "search.count.low";
    private final static String SEARCH_DEFAULT_COUNT_NAME = "search.count.default";
    private final static String SEARCH_MAX_PUBLIC_COUNT_NAME = "search.count.max.public";
    private final static String SEARCH_MAX_LOCALHOST_COUNT_NAME = "search.count.max.localhost";

    private final static int SEARCH_CACHE_THREASHOLD_TIME = 3000;

    private final static AtomicLong last_cache_search_time = new AtomicLong(10L);

    public final static String backend_hash = Integer.toHexString(Integer.MAX_VALUE);
    public final static String frontpeer_hash = Integer.toHexString(Integer.MAX_VALUE - 1);

    // possible values: cache, twitter, all
    public static TwitterTimeline search(final String[] protocolhostportstubs, final String query, final Set<String> filterList, final TwitterTimeline.Order order, final String source, final int count, final int timezoneOffset, final String provider_hash, final long timeout) throws IOException {
        String threadname = Thread.currentThread().getName();

        TwitterTimeline tl = new TwitterTimeline(order);
        if ("".equals(query)) return tl;
        String urlstring = "";
        String filterString = "";

        try {
            byte[] jsonb = null;
            IOException ee = null;
            backendloop: for (String protocolhostportstub: protocolhostportstubs) {
                Thread.currentThread().setName("searchOnOtherPeers on:" + protocolhostportstub);
                ee = null;
                try {
                    urlstring = protocolhostportstub + "/api/search.json?q=" + URLEncoder.encode(query.replace(' ', '+'), "UTF-8") + "&timezoneOffset=" + timezoneOffset + "&maximumRecords=" + count + "&source=" + (source == null ? "all" : source) + "&minified=true&shortlink=false&timeout=" + timeout;

                    if (!"".equals(filterString = String.join(", ", filterList))) {
                        urlstring = urlstring + "&filter=" + filterString;
                    }

                    jsonb = ClientConnection.download(urlstring);
                    break backendloop;
                } catch (IOException e) {
                    ee = e;
                }
            }
            if (jsonb == null || jsonb.length == 0) throw ee == null ? new IOException("empty content from " + protocolhostportstubs) : ee;
            String jsons = UTF8.String(jsonb);
            JSONObject json = new JSONObject(jsons);
            if (json == null || json.length() == 0) return tl;
            JSONArray statuses = json.getJSONArray("statuses");
            if (statuses != null) {
                for (int i = 0; i < statuses.length(); i++) {
                    JSONObject tweet = statuses.getJSONObject(i);
                    JSONObject user = tweet.getJSONObject("user");
                    if (user == null) continue;
                    tweet.remove("user");
                    UserEntry u = new UserEntry(user);
                    TwitterTweet t = new TwitterTweet(tweet);
                    tl.add(t, u);
                }
            }
            JSONObject metadata = json.getJSONObject("search_metadata");
            if (metadata != null) {
                Integer hits = metadata.has("hits") ? (Integer) metadata.get("hits") : null;
                if (hits != null) tl.setHits(hits.intValue());
                String scraperInfo = metadata.has("scraperInfo") ? (String) metadata.get("scraperInfo") : null;
                if (scraperInfo != null) tl.setScraperInfo(scraperInfo);
            }
        } catch (Throwable e) {
            //DAO.severe(e);
            throw new IOException(e.getMessage());
        }
        //System.out.println(parser.text());
        Thread.currentThread().setName(threadname);
        return tl;
    }

    public static TwitterTimeline search(
            final String[] protocolhostportstubs, final String query, final TwitterTimeline.Order order,
            final String source, final int count, final int timezoneOffset,
            final String provider_hash, final long timeout) throws IOException {
        return search(protocolhostportstubs, query, new HashSet<>(), order, source, count, timezoneOffset, provider_hash, timeout);
    }

    @Override
    protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
        doGet(request, response);
    }

    @Override
    protected void doGet(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
        final long start = System.currentTimeMillis();
        final Query post = RemoteAccess.evaluate(request);
        Map<String, String> inputMap = post.getMap();
        JSONObject metadata = new JSONObject(true);

        final ClientIdentity identity = AbstractAPIHandler.getIdentity(request, response, post);
        JSONArray dataArray = null;
        try {

            // manage DoS
            if (post.isDoS_blackout()) {response.sendError(503, "your (" + post.getClientHost() + ") request frequency is too high"); return;}

            // check call type
            String callType = "json";
            if(request.getServletPath().endsWith(".json")){
                callType = "json";
            } else if(request.getServletPath().endsWith(".rss")) {
                callType = "rss";
            } else if(request.getServletPath().endsWith(".txt")) {
                callType = "txt";
            }

            // evaluate get parameter
            String scraper = post.get("scraper", "");
            String callback = post.get("callback", "");
            boolean jsonp = callback != null && callback.length() > 0;
            boolean minified = post.get("minified", false);
            boolean shortlink_request = post.get("shortlink", true);

            // query parameters
            // naming schema according to SRU 2.0
            // see http://docs.oasis-open.org/search-ws/searchRetrieve/v1.0/os/part3-sru2.0/searchRetrieve-v1.0-os-part3-sru2.0.html
            // for compatibility "query" can be replaced with "q"
            String query = post.get("q", post.get("query", ""));
            query = CharacterCoding.html2unicode(query).replaceAll("\\+", " ");
            // paging options, syntax and meaning as defined by SRU 2.0
            final int startRecord = Math.max(1, post.get("startRecord", 1)); // the first record is number 1, not 0, see http://docs.oasis-open.org/search-ws/searchRetrieve/v1.0/os/part3-sru2.0/searchRetrieve-v1.0-os-part3-sru2.0.html#startAndMax
            final int maximumRecords = Math.min(
                    post.get("count", post.get("maximumRecords", (int) DAO.getConfig(SEARCH_DEFAULT_COUNT_NAME, 10))),
                    (int) (post.isDoS_servicereduction() ?
                            DAO.getConfig(SEARCH_LOW_COUNT_NAME, 10) :
                            post.isLocalhostAccess() ?
                                    DAO.getConfig(SEARCH_MAX_LOCALHOST_COUNT_NAME, 1000) :
                                    DAO.getConfig(SEARCH_MAX_PUBLIC_COUNT_NAME, 100)));

            String filter = post.get("filter", "").replaceAll("\\s","");
            Set<String> filterList = new HashSet<>();
            Arrays.asList(filter.split(",")).forEach(f -> filterList.add(f));

            // create tweet timeline
            final String ordername = post.get("order", Order.CREATED_AT.getMessageFieldName());
            final TwitterTimeline.Order order = TwitterTimeline.parseOrder(ordername);
            TwitterTimeline tl = DAO.timelineCache.getOrCreate(identity, query, startRecord <= 1, order);
            JSONObject hits = new JSONObject(true);
            final JSONObject[] aggregations = new JSONObject[]{null};

            if (!"".equals(scraper)) {
                // start a scraper
                inputMap.put("query", query);
                DAO.log(request.getServletPath() + " scraping with query: "
                        + query + " scraper: " + scraper);
                dataArray = DAO.scrapeLoklak(inputMap, true, true, metadata);
            } else if (tl.size() > 0) {
                // return the timeline from a cached search result
                // in case that the number of available records in the cache is too low, try to get more
                // otherwise there might be nothing to do here
            } else {
                final long timeout = (long) post.get("timeout", DAO.getConfig("search.timeout", 2000));
                String source = post.isDoS_servicereduction() ? "cache" : post.get("source", "all"); // possible values: cache, backend, twitter, all
                int agregation_limit = post.get("limit", 10);
                String[] fields = post.get("fields", new String[0], ",");
                int timezoneOffset = post.get("timezoneOffset", 0);
                if (query.indexOf("id:") >= 0 && ("all".equals(source) || "twitter".equals(source)))
                    source = "cache"; // id's cannot be retrieved from twitter with the scrape-api (yet), only from the cache

                final AtomicInteger cache_hits = new AtomicInteger(0);
                final AtomicInteger count_backend = new AtomicInteger(0);
                final AtomicInteger count_twitter_all = new AtomicInteger(0);
                final AtomicInteger count_twitter_new = new AtomicInteger(0);
                final boolean backend_push = DAO.getConfig("backend.push.enabled", false);
                final QueryEntry.Tokens tokens = new QueryEntry.Tokens(query);

                if ("all".equals(source)) {
                    // start all targets for search concurrently
                    final int timezoneOffsetf = timezoneOffset;
                    final String queryf = query;

                    // start a scraper
                    Thread scraperThread = tokens.raw.length() == 0 ? null : new Thread() {
                        public void run() {
                            final String scraper_query = tokens.translate4scraper();
                            DAO.log(request.getServletPath() + " scraping with query: " + scraper_query);
                            TwitterTimeline twitterTl = DAO.scrapeTwitter(post, filterList, scraper_query, order, timezoneOffsetf, true, timeout, true);
                            count_twitter_new.set(twitterTl.size());
                            tl.putAll(QueryEntry.applyConstraint(twitterTl, tokens, false)); // pre-localized results are not filtered with location constraint any more
                            tl.setScraperInfo(twitterTl.getScraperInfo());
                            post.recordEvent("twitterscraper_time", System.currentTimeMillis() - start);
                        }
                    };
                    if (scraperThread != null) scraperThread.start();

                    // start a local search
                    Thread localThread = queryf == null || queryf.length() == 0 ? null : new Thread() {
                        public void run() {

                            DAO.SearchLocalMessages localSearchResult =
                                    new DAO.SearchLocalMessages(
                                            queryf,
                                            order,
                                            timezoneOffsetf,
                                            last_cache_search_time.get() > SEARCH_CACHE_THREASHOLD_TIME ? Math.min(maximumRecords,
                                            (int) DAO.getConfig(SEARCH_LOW_COUNT_NAME, 10)) : maximumRecords,
                                            agregation_limit,
                                            filterList,
                                            fields
                                    );
                            long time = System.currentTimeMillis() - start;
                            last_cache_search_time.set(time);
                            post.recordEvent("cache_time", time);
                            cache_hits.set(localSearchResult.timeline.getHits());
                            tl.putAll(localSearchResult.timeline);
                            tl.setResultIndex(localSearchResult.timeline.getResultIndex());
                            aggregations[0] = localSearchResult.getAggregations();
                        }
                    };
                    if (localThread != null) localThread.start();

                    // start a backend search, but only if backend_push == true or result from scraper is too bad
                    boolean start_backend_thread = false;
                    if (backend_push) start_backend_thread = true;
                    else {
                        // wait now for termination of scraper thread and local search
                        // to evaluate how many results are available
                        if (scraperThread != null) try {
                            scraperThread.join(Math.max(10000, timeout - System.currentTimeMillis() + start));
                        } catch (InterruptedException e) {
                        }
                        if (localThread != null) try {
                            localThread.join(Math.max(100, timeout - System.currentTimeMillis() + start));
                        } catch (InterruptedException e) {
                        }
                        localThread = null;
                        scraperThread = null;
                        if (tl.size() < maximumRecords) start_backend_thread = true;
                    }
                    Thread backendThread = tokens.original.length() == 0 || !start_backend_thread ? null : new Thread() {
                        public void run() {
                            TwitterTimeline backendTl = DAO.searchBackend(tokens.original, filterList, order, maximumRecords, timezoneOffsetf, "cache", timeout);
                            if (backendTl != null) {
                                tl.putAll(QueryEntry.applyConstraint(backendTl, tokens, true));
                                count_backend.set(tl.size());
                                // TODO: read and aggregate aggregations from backend as well
                            }
                            post.recordEvent("backend_time", System.currentTimeMillis() - start);
                        }
                    };
                    if (backendThread != null) backendThread.start();

                    // wait for termination of all threads
                    if (scraperThread != null) try {
                        scraperThread.join(Math.max(10000, timeout - System.currentTimeMillis() + start));
                    } catch (InterruptedException e) {
                    }

                    // in case that the scraper thread had been started and was successful, we do not wait for the other threads to terminate
                    if (scraperThread == null || tl.getHits() == 0 || query.indexOf(':') >= 0 || query.indexOf('/') >= 0 || fields.length > 0) {
                        if (localThread != null) try {
                            localThread.join(Math.max(100, timeout - System.currentTimeMillis() + start));
                        } catch (InterruptedException e) {
                        }
                    }
                    if (scraperThread == null || tl.getHits() == 0) {
                        if (backendThread != null) try {
                            backendThread.join(Math.max(100, timeout - System.currentTimeMillis() + start));
                        } catch (InterruptedException e) {
                        }
                    }
                } else if ("twitter".equals(source) && tokens.raw.length() > 0) {
                    final String scraper_query = tokens.translate4scraper();
                    DAO.log(request.getServletPath() + " scraping with query: " + scraper_query);
                    TwitterTimeline twitterTl = DAO.scrapeTwitter(post, filterList, scraper_query, order, timezoneOffset, true, timeout, true);

                    count_twitter_new.set(twitterTl.size());
                    tl.putAll(QueryEntry.applyConstraint(twitterTl, tokens, false)); // pre-localized results are not filtered with location constraint any more
                    tl.setScraperInfo(twitterTl.getScraperInfo());
                    post.recordEvent("twitterscraper_time", System.currentTimeMillis() - start);
                    // in this case we use all tweets, not only the latest one because it may happen that there are no new and that is not what the user expects

                } else if ("cache".equals(source)) {
                    DAO.SearchLocalMessages localSearchResult =
                            new DAO.SearchLocalMessages(
                                    query,
                                    order,
                                    timezoneOffset,
                                    last_cache_search_time.get() > SEARCH_CACHE_THREASHOLD_TIME
                                            ? Math.min(maximumRecords,(int) DAO.getConfig(SEARCH_LOW_COUNT_NAME, 10))
                                            : maximumRecords,
                                    agregation_limit,
                                    filterList,
                                    fields
                            );
                    cache_hits.set(localSearchResult.timeline.getHits());
                    tl.putAll(localSearchResult.timeline);
                    tl.setResultIndex(localSearchResult.timeline.getResultIndex());
                    aggregations[0] = localSearchResult.getAggregations();
                    long time = System.currentTimeMillis() - start;
                    last_cache_search_time.set(time);
                    post.recordEvent("cache_time", time);

                } else if ("backend".equals(source) && query.length() > 0) {
                    TwitterTimeline backendTl = DAO.searchBackend(query, filterList, order, maximumRecords, timezoneOffset, "cache", timeout);
                    if (backendTl != null) {
                        tl.putAll(QueryEntry.applyConstraint(backendTl, tokens, true));
                        tl.setScraperInfo(backendTl.getScraperInfo());
                        // TODO: read and aggregate aggregations from backend as well
                        count_backend.set(tl.size());
                    }
                    post.recordEvent("backend_time", System.currentTimeMillis() - start);

                }

                // check the latest user_ids
                DAO.announceNewUserId(tl);

                hits.put("count_twitter_all", count_twitter_all.get());
                hits.put("count_twitter_new", count_twitter_new.get());
                hits.put("count_backend", count_backend.get());
                hits.put("cache_hits", cache_hits.get());
            }

            // create json or xml according to path extension
            int shortlink_iflinkexceedslength = shortlink_request ? (int) DAO.getConfig("shortlink.iflinkexceedslength", 500L) : Integer.MAX_VALUE;
            String shortlink_urlstub = DAO.getConfig("shortlink.urlstub", "http://127.0.0.1:9000");

            dataOutput( response, dataArray, query, startRecord, maximumRecords, minified,
                    callType, jsonp, post, tl, aggregations[0], hits, callback, shortlink_urlstub,
                    shortlink_iflinkexceedslength, filter, metadata);

            post.recordEvent("result_count", tl.size());
            post.recordEvent("postprocessing_time", System.currentTimeMillis() - start);
            post.recordEvent("hits", hits);
            DAO.log(request.getServletPath() + "?" + request.getQueryString()
                    + " -> " + tl.size() + " records returned");
            post.finalize();
        } catch (Throwable e) {
            DAO.severe(e.getMessage(), e);
            //DAO.severe(e);
        }
    }

    private static void dataOutput(HttpServletResponse response, JSONArray dataArray,
            String query, int startRecord, int maximumRecords, boolean minified,
            String callType, boolean jsonp, Query post, TwitterTimeline tl, JSONObject aggregations,
            JSONObject hits, String callback, String shortlink_urlstub,
            int shortlink_iflinkexceedslength, String filter, JSONObject metadata) throws IOException {
        if ("json".equals(callType)) {
            post.setResponse(response, jsonp ? "application/javascript" : "application/json");

            JSONObject m = new JSONObject(true);
            JSONArray statuses = new JSONArray();

            try {
                for (TwitterTweet t : tl.getNextTweets(startRecord - 1, maximumRecords)) {
                    UserEntry u = tl.getUser(t);
                    statuses.put(t.toJSON(u, true, shortlink_iflinkexceedslength, shortlink_urlstub));
                }
            } catch (ConcurrentModificationException e) {
                // late incoming messages from concurrent peer retrieval may cause this
                // we silently do nothing here and return what we listed so far
            }

            if (!minified) setReadMe(m);
            if (dataArray == null) {
                setTwitterMetaData(metadata, startRecord, maximumRecords, tl, query, filter, post, hits);
                m.put("search_metadata", metadata);
                m.put("statuses", statuses);
            } else {
                moreMetaData(metadata, post);
                m.put("results", dataArray);
                m.put("metadata", metadata);
            }
            m.put("aggregations", aggregations);

            // write json
            response.setCharacterEncoding("UTF-8");
            PrintWriter sos = response.getWriter();
            if (jsonp) sos.print(callback + "(");
            sos.print(m.toString(minified ? 0 : 2));
            if (jsonp) sos.println(");");
            sos.println();
        } else if ("rss".equals(callType)) {
            response.setCharacterEncoding("UTF-8");
            post.setResponse(response, "application/rss+xml;charset=utf-8");

            // generate xml
            RSSMessage channel = new RSSMessage();
            channel.setPubDate(new Date());
            channel.setTitle("RSS feed for Twitter search for " + query);
            channel.setDescription("");
            channel.setLink("");
            RSSFeed feed = new RSSFeed(tl.size());
            feed.setChannel(channel);

            try {
                for (TwitterTweet t : tl.getNextTweets(startRecord - 1, maximumRecords)) {
                    UserEntry u = tl.getUser(t);
                    RSSMessage m = new RSSMessage();
                    m.setLink(t.getStatusIdUrl().toExternalForm());
                    m.setAuthor(u.getName() + " @" + u.getScreenName());
                    m.setTitle(u.getName() + " @" + u.getScreenName());
                    m.setDescription(t.moreData.getText(shortlink_iflinkexceedslength, shortlink_urlstub, t.getText(), t.getLinks(), t.getPostId()).text);
                    m.setPubDate(t.getCreatedAt());
                    m.setGuid(t.getPostId());
                    feed.addMessage(m);
                }
            } catch (ConcurrentModificationException e) {
                // late incoming messages from concurrent peer retrieval may cause this
                // we silently do nothing here and return what we listed so far
            }
            String rss = feed.toString();
            //System.out.println("feed has " + feed.size() + " entries");

            // write xml
            response.getOutputStream().write(UTF8.getBytes(rss));
        } else if ("txt".equals(callType)) {
            post.setResponse(response, "text/plain");
            final StringBuilder buffer = new StringBuilder(1000);

            // Write text file
            try {
                for (TwitterTweet t : tl) {
                    UserEntry u = tl.getUser(t);
                    buffer.append(t.getCreatedAt()).append(" ")
                            .append(u.getScreenName())
                            .append(": ")
                            .append(t.moreData.getText(
                                    shortlink_iflinkexceedslength, shortlink_urlstub,
                                    t.getText(), t.getLinks(), t.getPostId()).text)
                            .append('\n');
                }
            } catch (ConcurrentModificationException e) {
                // late incoming messages from concurrent peer retrieval may cause this
                // we silently do nothing here and return what we listed so far
            }
            response.getOutputStream().write(UTF8.getBytes(buffer.toString()));
        }
    }

    private static void setReadMe(JSONObject m) {
        m.put("readme_0", "THIS JSON IS THE RESULT OF YOUR SEARCH QUERY - THERE IS NO WEB PAGE WHICH SHOWS THE RESULT!");
        m.put("readme_1", "loklak.org is the framework for a message search system, not the portal, read: http://api.loklak.org/about.html#notasearchportal");
        m.put("readme_2", "This is supposed to be the back-end of a search portal. For the api, see http://api.loklak.org/api.html");
        m.put("readme_3", "Parameters q=(query), source=(cache|backend|twitter|all), callback=p for jsonp, maximumRecords=(message count), minified=(true|false)");
    }

    private static void moreMetaData(JSONObject metadata, Query post) {
        metadata.put("client", post.getClientHost());
        metadata.put("time", System.currentTimeMillis() - post.getAccessTime());
        metadata.put("servicereduction", post.isDoS_servicereduction() ? "true" : "false");
//        metadata.put("scraper", scraperInfo);
//        String index = tl.getResultIndex() == null ? "" : String.valueOf(tl.getResultIndex());
//        metadata.put("index", index);
    }

    private static void setTwitterMetaData(JSONObject metadata, final int startRecord,
            final int maximumRecords, TwitterTimeline tl, String query, String filter, Query post, JSONObject hits) {
        // the number of the first record (according to SRU set to 1 for very first)
        metadata.put("startRecord", startRecord);
        // number of records within this json result set returned in the api call
        metadata.put("maximumRecords", maximumRecords);
        // number of records available in the search cache (so far, may be increased later > hits)
        metadata.put("count", tl.size());
        // number of records in the search index (so far, may be increased later as well)
        metadata.put("hits", tl.getHits());
        if (tl.getOrder() == Order.CREATED_AT) metadata.put("period", tl.period());
        metadata.put("query", query);
        metadata.put("filter", filter);
        metadata.put("client", post.getClientHost());
        metadata.put("time", System.currentTimeMillis() - post.getAccessTime());
        metadata.put("servicereduction", post.isDoS_servicereduction() ? "true" : "false");
        metadata.putAll(hits);

        String scraperInfo = tl.getScraperInfo() == null ? "" : tl.getScraperInfo();
        metadata.put("scraperInfo", scraperInfo);
        String index = tl.getResultIndex() == null ? "" : String.valueOf(tl.getResultIndex());
        metadata.put("index", index);
    }

    public static void main(String[] args) {
        try {
            TwitterTimeline tl = search(new String[]{"http://root.loklak.org"}, "beer", Order.CREATED_AT, "cache", 20, -120, backend_hash, 10000);
            System.out.println(tl.toJSON(false, "search_metadata", "statuses").toString(2));
        } catch (IOException e) {
            DAO.severe(e);
        }
    }
} public class Post extends JSONObject implements ObjectEntry {

    /**
     * timestamp is when the data was scraped in int datatype
     */
    protected long timestamp = 0;

    /**
     * createdAt is date when the data in Post Object was created in source. If there isn't any
     * value, it is set to timestamp.
     */
    private Date createdAt = null;

    /**
     * Unique id for Post data object.
     */
    protected String postId = null;

    /**
     * Whether Post data object is wrapper of Post objects or not. By default, it is set to false.
     */
    private boolean wrapper = false;

    /**
     * Default constructor with timestamp of Post object created
     */
    public Post() {
        super(true);
        this.setTimestamp();
    }

    /**
     * This is a wrapper Post constructor. This acts as wrapper for Post data objects. This object
     * doesn't have post-id or any timestamp.
     *
     * @param wrapper
     *          if wrapper=true, it is a wrapper else it isn't.
     */
    public Post(boolean wrapper) {
        super(true);
        if(!wrapper) {
            this.setTimestamp();
        } else {
            this.wrapper = true;
        }
    }

    /**
     * Post constructor to convert JSONObject data to Post object.
     *
     * @param json
     *          JSON data as input to Post
     */
    public Post(JSONObject json) {
        super(json.toString());
    }

    /**
     * Post constructor to convert Map to Post object.
     *
     * @param map
     *          A map with key as String and value as Object
     */
    public Post(Map<String, Object> map) {
        super(map);
    }

    /**
     * Post constructor to convert json data in string to Post object.
     *
     * @param data
     *          A string data in json format
     * @param query
     *          Query value to set it as post-id
     */
    public Post(String data, String query) {
        super(data);
        this.setTimestamp();
        this.setPostId(query);
    }

    /**
     * Post constructor with input value of timestamp.
     *
     * @param timestamp
     *          Timestamp at which data was scraped.
     */
    protected Post(long timestamp) {
        this.setTimestamp(timestamp);
    }

    /**
     * To check if this Post object is wrapper or not.
     *
     * @return
     *          A boolean value, true if this is wrapper, else returns false.
     */
    public boolean isWrapper() {
        return this.wrapper;
    }

    /**
     * Returns Post data object's data to JSON string
     *
     * @return
     *          Data as JSON string
     */
    public String toString() {
        return super.toString();
    }

    /**
     * Returns Post data object's data to JSONObject
     *
     * @return
     *          Data as JSONObject
     */
    public JSONObject toJSON() {
        return this;
    }

    /**
     * Channels on which the Tweet will be published - all (default)
     *
     * @return
     *          Array of channels to publish message to.
     */
    protected String[] getStreamChannels() {
        return new String[] {"all"};
    }

    /**
     * Publish data to MQTT from Post data object
     */
    public final void publishToMQTT() {
        if (DAO.mqttPublisher != null) {
            // Will be null if stream is disabled
            DAO.mqttPublisher.publish(this.getStreamChannels(), this.toString());
        }
    }

    public void setTimestamp(long timestamp) {
        this.put("timestamp_id", timestamp);
        this.timestamp = timestamp;
    }

    public void setTimestamp() {
        if(this.has("timestamp_id")) {
            long timestampValue = Long.parseLong(String.valueOf(this.get("timestamp_id")));
            this.setTimestamp(timestampValue);
        } else {
            long timestamp = System.currentTimeMillis();
            this.setTimestamp(timestamp);
        }
    }

    /**
     * a time stamp that is given in loklak upon the arrival of the tweet which is the current local time
     * @return
     */
    public long getTimestamp() {
        if(this.timestamp == 0 && this.has("timestamp_id")) {
            this.timestamp = Long.parseLong(String.valueOf(this.get("timestamp_id")));
        }
        this.setTimestamp(this.timestamp);
        return this.timestamp;
    }

    public Date getTimestampDate() {
        DAO.log("date is " + String.valueOf(this.getTimestamp()) + "    " + String.valueOf(new Date(this.getTimestamp())));
        return new Date(this.getTimestamp());
    }

    public DateTime getTimestampDate(String a) {
        DAO.log("date is " + String.valueOf(this.getTimestamp()) + "    " + String.valueOf(new Date(this.getTimestamp())));
        return new DateTime(this.getTimestamp(), DateTimeZone.UTC);
    }

    public void setCreated(Date _createdAt) {
        this.createdAt = _createdAt;
    }

    /**
     * the time given in the tweet which is the time when the user created it.
     * This is also use to do the index partition into minute, hour, week
     * @return
     */
    public Date getCreated() {
        if(this.createdAt == null) {
            this.createdAt = this.getTimestampDate();
        }
        return this.createdAt;
    }

    private void setPostId() {
        if(this.has("id_str")) {
            this.setPostId(String.valueOf(this.get("id_str")));
        } else {
            this.setPostId(String.valueOf(this.getTimestamp()));
        }
    }

    public void setPostId(String postId) {
        this.put("id_str", postId);
        this.postId = postId;
    }

    public String getPostId() {
        if(this.has("id_str")) {
            this.postId = String.valueOf(this.get("id_str"));
        }
        return this.postId;
    }
} public class TwitterScraper {

    public static final ExecutorService executor = Executors.newFixedThreadPool(40);
    public static final Pattern emoji_pattern_span = Pattern.compile("<span [^>]*class=\"Emoji Emoji--forLinks\" [^>]*>[\\n]*[^<]*</span>[\\n]*<span [^>]*class=\"visuallyhidden\" [^>]*aria-hidden=\"true\"[^>]*>[\\n]*([^<]*)[\\n]*</span>");

    public static TwitterTimeline search(
            final String query,
            final Set<String> filterList,
            final TwitterTimeline.Order order,
            final boolean writeToIndex,
            final boolean writeToBackend,
            int jointime) {
        TwitterTimeline[] tl = search(query, filterList, order, writeToIndex, writeToBackend);
        long timeout = System.currentTimeMillis() + jointime;
        long remainingWait = 0;
        for (TwitterTweet tt: tl[1]) {
            remainingWait = Math.max(10, timeout - System.currentTimeMillis());
            if (tt.waitReady(remainingWait)) {
                 // double additions are detected
                tl[0].add(tt, tt.getUser());
            }
        }
        return tl[0];
    }

    public static TwitterTimeline search(
            final String query,
            final TwitterTimeline.Order order,
            final boolean writeToIndex,
            final boolean writeToBackend,
            int jointime) {

        return search(query, new HashSet<>(), order, writeToIndex, writeToBackend, jointime);
    }

    private enum FilterType {
        tweets, image, video, geo, links, mentions, hashtags, media;
    }
    
    private static String prepareSearchUrl(final String query, FilterType type) {
        // check https://twitter.com/search-advanced for a better syntax
        // https://support.twitter.com/articles/71577-how-to-use-advanced-twitter-search#

        // query q
        StringBuilder t = new StringBuilder(query.length());
        for (String s: query.replace('+', ' ').split(" ")) {
            t.append(' ');
            if (s.startsWith("since:") || s.startsWith("until:")) {
                int u = s.indexOf('_');
                t.append(u < 0 ? s : s.substring(0, u));
            } else {
                t.append(s);
            }
        }
        if (type == FilterType.image && t.indexOf("has:images") < 0) t.append(" has:images");
        if (type == FilterType.video && t.indexOf("has:videos") < 0) t.append(" has:videos");
        
        String q;
        try {
            q = t.length() == 0 ? "*" : URLEncoder.encode(t.substring(1), "UTF-8");
        } catch (UnsupportedEncodingException e) {
            q = t.length() == 0 ? "*" : t.substring(1);
        }

        // building url
        // see https://github.com/bisguzar/twitter-scraper/blob/master/twitter_scraper/modules/tweets.py#L12-L29
        return "https://twitter.com/i/search/timeline?f=tweets&vertical=default&q=" + q + "&src=typd&reset_error_state=false&include_available_features=1&include_entities=1&include_new_items_bar=true";
    }

    private static TwitterTimeline[] search(
            final String query,
            final Set<String> filterList,
            final TwitterTimeline.Order order,
            final boolean writeToIndex,
            final boolean writeToBackend) {
        // check
        // https://twitter.com/search-advanced for a better syntax
        // https://support.twitter.com/articles/71577-how-to-use-advanced-twitter-search#
        FilterType tt = FilterType.tweets;
        if (filterList.contains("image")) tt = FilterType.image;
        if (filterList.contains("video")) tt = FilterType.video;
        String https_url = prepareSearchUrl(query, tt);
        TwitterTimeline[] timelines = null;
        try {
            ClientConnection connection = new ClientConnection(https_url, query);
            if (connection.getInputStream() == null) return null;
            try {
                BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));

                JSONTokener t = new JSONTokener(br);
                JSONObject j = new JSONObject(t);
                String html = j.getString("items_html");
                Document doc = Jsoup.parse(html);
                
                timelines = search(doc, filterList, order, writeToIndex, writeToBackend);
            } catch (IOException e) {
                DAO.severe(e);
            } finally {
                connection.close();
            }
            /*
            HtmlUnitLoader loader = new HtmlUnitLoader(https_url, "scraper");
            ByteArrayInputStream bais = new ByteArrayInputStream(loader.getXml().getBytes(StandardCharsets.UTF_8));
            BufferedReader br = new BufferedReader(new InputStreamReader(bais, StandardCharsets.UTF_8));
            timelines = search(br, filterList, order, writeToIndex, writeToBackend);
            */
        } catch (IOException e) {
            // this could mean that twitter rejected the connection (DoS protection?) or we are offline (we should be silent then)
            // DAO.severe(e);
            if (timelines == null) timelines = new TwitterTimeline[]{new TwitterTimeline(order), new TwitterTimeline(order)};
        };

        // wait until all messages in the timeline are ready
        if (timelines == null) {
            // timeout occurred
            timelines = new TwitterTimeline[]{new TwitterTimeline(order), new TwitterTimeline(order)};
        }
        if (timelines != null) {
            if (timelines[0] != null) timelines[0].setScraperInfo("local");
            if (timelines[1] != null) timelines[1].setScraperInfo("local");
        }
        return timelines;
    }

    /**
     * scrape messages from the reader stream: this already checks if a message is new. There are only new messages returned
     * @param br
     * @param order
     * @return two timelines in one array: Timeline[0] is the one which is finished to be used, Timeline[1] contains messages which are in postprocessing
     * @throws IOException
     */
    private static TwitterTimeline[] search(
            final Document doc,
            final Set<String> filterList,
            final TwitterTimeline.Order order,
            final boolean writeToIndex,
            final boolean writeToBackend) throws IOException {
        TwitterTimeline timelineReady = new TwitterTimeline(order);
        TwitterTimeline timelineWorking = new TwitterTimeline(order);
        Set<String> images = null;
        Set<String> videos = null;
        String place_id = "";
        String place_name = "";
        Set<String> mentions = new LinkedHashSet<>();
        boolean debuglog = DAO.getConfig("flag.debug.twitter_scraper", "false").equals("true");

        // parse
        Elements items = doc.getElementsByClass("stream-item");
        itemloop: for (int itemc = 0; itemc < items.size(); itemc++) {
            Element item = items.get(itemc);
            if (debuglog) System.out.println(item.toString());
    
            Elements streamitem = item.getElementsByClass("stream-item");
            Element tweeti = streamitem.get(0).child(0);
            String tweetID = tweeti.attr("data-tweet-id");
            String conversationID = tweeti.attr("data-conversation-id"); // this is the parent in a conversation
            String mentions_screenname = tweeti.attr("data-mentions"); // the conversationID does not have a link to the user name which produces the actual tweet link. This here is a hint. But several names can be listed here.
            // the first nam in mentions_screenname is the latest in the conversation chain
            Set<String> mentions_screennames = new LinkedHashSet<>(); // to preserve the order we use a linked hash set
            String[] mss= mentions_screenname.split(" ");
            for (int i = 0; i < mss.length; i++) mentions_screennames.add(mss[i]);
            if (conversationID != null && conversationID.equals(tweetID)) {conversationID = ""; mentions_screennames.clear();}
            
            Elements profile = item.getElementsByClass("js-profile-popup-actionable");
            Elements avatare = item.getElementsByClass("js-action-profile-avatar");
            
            String userid = profile.attr("data-user-id");
            String usernickname = profile.attr("data-screen-name");
            String useravatarurl = avatare.attr("src");
            String userfullname = profile.attr("data-name");
            
            UserEntry user = new UserEntry(
                    userid,
                    usernickname,
                    useravatarurl,
                    MessageEntry.html2utf8(userfullname)
            );
    
            Elements timestamp = item.getElementsByClass("_timestamp");
            String rawtext = item.getElementsByClass("tweet-text").text();
            if (mentions_screennames.size() > 0) {
                // attention: this turns the order around! First entry will be conversation starter!
                for (String screenname: mentions_screennames) {
                    if (rawtext.indexOf(screenname) < 0) rawtext = "@" + screenname + " " + rawtext;
                }
            }
            
            String tweettimes = timestamp.attr("data-time-ms");
            if (tweettimes.length() == 0) continue itemloop; // sometimes tweets are not available any more
            long tweettime = Long.parseLong(tweettimes);
            // this assertion holds in most, but unfortunately not in all cases, so it is commented out
            // long snowflaketime = snowflake2millis(Long.parseLong(tweetID));
            // assert tweettime / 1000L == snowflaketime / 1000L : "tweettime = " + tweettime + ", snowflaketime = " + snowflaketime;

            Elements reply = item.getElementsByClass("ProfileTweet-action--reply").get(0).children();
            Elements retweet = item.getElementsByClass("ProfileTweet-action--retweet").get(0).children();
            Elements favourite = item.getElementsByClass("ProfileTweet-action--favorite").get(0).children();
            String tweetreplycount = reply.attr("data-tweet-stat-count");
            String tweetretweetcount = retweet.attr("data-tweet-stat-count");
            String tweetfavouritecount = favourite.attr("data-tweet-stat-count");
            
            String tweetstatusurl = item.getElementsByClass("tweet-timestamp").attr("href");        
            // evaluate
            
            // patch text with non-mentioned mentions (twitter obviously cuts that away)
            mentions.remove(user.getScreenName());
            for (String mention: mentions) {
                String m = "@" + mention;
                if (rawtext.indexOf(m) < 0) rawtext = rawtext + " " + m;
            }
            
            TwitterTweet tweet;
            try {
                tweet = new TwitterTweet(
                        user.getScreenName(),
                        tweettime,
                        tweetID, // like 1284585691259850753
                        conversationID, // like 1284532773563359233
                        mentions_screennames,
                        tweetstatusurl, // like /yacy_search/status/1284585691259850753
                        rawtext,
                        Long.parseLong(tweetreplycount),
                        Long.parseLong(tweetretweetcount),
                        Long.parseLong(tweetfavouritecount),
                        images, videos, place_name, place_id,
                        user, writeToIndex, writeToBackend
                );
    
                place_id = "";
                place_name = "";
                mentions.clear();
                videos = null;
                images = null;
            } catch (NullPointerException e) {
                DAO.severe(e);
                tweet = null;
            }
            
            //if (DAO.messages == null || !DAO.messages.existsCache(tweet.getPostId())) {
                // checking against the exist cache is incomplete. A false negative would just cause that a tweet is
                // indexed again.
                if (tweet.willBeTimeConsuming()) {
                    executor.execute(tweet);
                    //new Thread(tweet).start();
                    // because the executor may run the thread in the current thread it could be possible that the result is here already
                    if (tweet.isReady()) {
    
                        timelineReady.add(tweet, user);
                        //DAO.log("SCRAPERTEST: messageINIT is ready");
                    } else {
                        timelineWorking.add(tweet, user);
                        //DAO.log("SCRAPERTEST: messageINIT unshortening");
                    }
                } else {
                    // no additional thread needed, run the postprocessing in the current thread
                    tweet.run();
                    timelineReady.add(tweet, user);
                }
            //}
        }

        return new TwitterTimeline[]{timelineReady, timelineWorking};
    }

    private static long snowflake2millis(long sf) {
        return (sf >> 22) + 1288834974657L;
    }
    
    private final static Pattern hashtag_pattern = Pattern.compile("<a href=\"/hashtag/.*?\".*?class=\"twitter-hashtag.*?\".*?><s>#</s><b>(.*?)</b></a>");
    private final static Pattern timeline_link_pattern = Pattern.compile("<a href=\"https://(.*?)\".*? data-expanded-url=\"(.*?)\".*?twitter-timeline-link.*?title=\"(.*?)\".*?>.*?</a>");
    private final static Pattern timeline_embed_pattern = Pattern.compile("<a href=\"(https://t.co/\\w+)\" class=\"twitter-timeline-link.*?>pic.twitter.com/(.*?)</a>");
    private final static Pattern emoji_pattern = Pattern.compile("<img .*?class=\"Emoji Emoji--forText\".*?alt=\"(.*?)\".*?>");
    private final static Pattern doublespace_pattern = Pattern.compile("  ");
    private final static Pattern cleanup_pattern = Pattern.compile(
            "</?(s|b|strong)>|" +
                    "<a href=\"/hashtag.*?>|" +
                    "<a.*?class=\"twitter-atreply.*?>|" +
                    "<span.*?span>"
    );

    public static class TwitterTweet extends Post implements Runnable {

        public final Semaphore ready;
        public MessageEntry moreData = new MessageEntry();
        public UserEntry user;
        public boolean writeToIndex;
        public boolean writeToBackend;

        // a time stamp that is given in loklak upon the arrival of the tweet which is the current local time
        public Date timestampDate;
        // the time given in the tweet which is the time when the user created it.
        // This is also use to do the index partition into minute, hour, week
        public Date created_at;
        // on means 'valid from'
        public Date on;
        // 'to' means 'valid_until' and may not be set
        public Date to;

        // where did the message come from
        protected SourceType source_type;
        // who created the message
        protected ProviderType provider_type;

        public String provider_hash, screen_name, retweet_from, postId, conversationID, canonical_id, parent, text;
        public Set<String> conversationUserIDs;
        protected URL status_id_url;
        protected long reply_count, retweet_count, favourite_count;
        public Set<String> images, audios, videos;
        protected String place_name, place_id;

        // the following fields are either set as a common field or generated by extraction from field 'text' or from field 'place_name'
        // coordinate order is [longitude, latitude]
        protected double[] location_point, location_mark;
        // Value in metres
        protected int location_radius;
        protected LocationSource location_source;
        protected PlaceContext place_context;
        protected String place_country;

        // The length of tweets without links, users, hashtags
        // the following can be computed from the tweet data but is stored in the search index
        // to provide statistical data and ranking attributes
        private int without_l_len, without_lu_len, without_luh_len;

        // the arrays of links, users, hashtags
        private List<String> users, hosts, links, mentions, hashtags;

        private boolean enriched;

        public TwitterTweet(
                final String user_screen_name_raw,
                final long created_at_raw,
                final String tweetID, // the ID of the tweet
                final String tweetConversationID, // the ID of a tweet where this is a reply-to tweet
                final Set<String> mentions_screennames,
                final String status_id_url_raw,
                final String text_raw,
                final long replies,
                final long retweets,
                final long favourites,
                final Set<String> images,
                final Set<String> videos,
                final String place_name,
                final String place_id,
                final UserEntry user,
                final boolean writeToIndex,
                final boolean writeToBackend) throws MalformedURLException {
            super();
            this.source_type = SourceType.TWITTER;
            this.provider_type = ProviderType.SCRAPED;
            this.screen_name = user_screen_name_raw;
            this.created_at = new Date(created_at_raw);
            this.status_id_url = new URL("https://twitter.com" + status_id_url_raw);
            int p = status_id_url_raw.lastIndexOf('/');
            this.postId = p >= 0 ? status_id_url_raw.substring(p + 1) : "-1";
            assert this.postId.equals(tweetID);
            this.conversationID = tweetConversationID;
            this.conversationUserIDs = mentions_screennames;
            this.reply_count = replies;
            this.retweet_count = retweets;
            this.favourite_count = favourites;
            this.place_name = place_name;
            this.place_id = place_id;
            this.images = images;
            this.videos = videos;
            this.text = text_raw;
            this.user = user;
            this.writeToIndex = writeToIndex;
            this.writeToBackend = writeToBackend;

            //Date d = new Date(timemsraw);
            //System.out.println(d);

            /* failed to reverse-engineering the place_id :(
            if (place_id.length() == 16) {
                String a = place_id.substring(0, 8);
                String b = place_id.substring(8, 16);
                long an = Long.parseLong(a, 16);
                long bn = Long.parseLong(b, 16);
                System.out.println("place = " + place_name + ", a = " + an + ", b = " + bn);
                // Frankfurt a = 3314145750, b = 3979907708, http://www.openstreetmap.org/#map=15/50.1128/8.6835
                // Singapore a = 1487192992, b = 3578663936
            }
            */

            // this.text MUST be analysed with analyse(); this is not done here because it should be started concurrently; run run();

            this.ready = new Semaphore(0);
        }

        public TwitterTweet(JSONObject json) {
            this.moreData = new MessageEntry();
            Object timestamp_obj = lazyGet(json, AbstractObjectEntry.TIMESTAMP_FIELDNAME);
            this.timestampDate = MessageEntry.parseDate(timestamp_obj);
            this.timestamp = this.timestampDate.getTime();
            Object created_at_obj = lazyGet(json, AbstractObjectEntry.CREATED_AT_FIELDNAME);
            this.created_at = MessageEntry.parseDate(created_at_obj);
            Object on_obj = lazyGet(json, "on");
            this.on = on_obj == null ? null : MessageEntry.parseDate(on);
            Object to_obj = lazyGet(json, "to");
            this.to = to_obj == null ? null : MessageEntry.parseDate(to);
            String source_type_string = (String) lazyGet(json, "source_type");
            try {
                this.source_type = source_type_string == null ? SourceType.GENERIC : SourceType.byName(source_type_string);
            } catch (IllegalArgumentException e) {
                this.source_type = SourceType.GENERIC;
            }
            String provider_type_string = (String) lazyGet(json, "provider_type");
            if (provider_type_string == null) provider_type_string = ProviderType.NOONE.name();
            try {
                this.provider_type = ProviderType.valueOf(provider_type_string);
            } catch (IllegalArgumentException e) {
                this.provider_type = ProviderType.NOONE;
            }
            this.provider_hash = (String) lazyGet(json, "provider_hash");
            this.screen_name = (String) lazyGet(json, "screen_name");
            this.retweet_from = (String) lazyGet(json, "retweet_from");
            this.postId = (String) lazyGet(json, "id_str");
            this.conversationID = (String) lazyGet(json, "conversationid_str");
            this.conversationUserIDs = MessageEntry.parseArrayList(lazyGet(json, "conversation_user"));
            this.text = (String) lazyGet(json, "text");
            try {
                this.status_id_url = new URL((String) lazyGet(json, "link"));
            } catch (MalformedURLException e) {
                this.status_id_url = null;
            }
            this.reply_count = MessageEntry.parseLong((Number) lazyGet(json, "reply_count"));
            this.retweet_count = MessageEntry.parseLong((Number) lazyGet(json, "retweet_count"));
            this.favourite_count = MessageEntry.parseLong((Number) lazyGet(json, "favourites_count")); // inconsitency in naming, but twitter api defines so
            this.images = MessageEntry.parseArrayList(lazyGet(json, "images"));
            this.audios = MessageEntry.parseArrayList(lazyGet(json, "audio"));
            this.videos = MessageEntry.parseArrayList(lazyGet(json, "videos"));
            this.place_id = MessageEntry.parseString((String) lazyGet(json, "place_id"));
            this.place_name = MessageEntry.parseString((String) lazyGet(json, "place_name"));
            this.place_country = MessageEntry.parseString((String) lazyGet(json, "place_country"));

            if (this.place_country != null && this.place_country.length() != 2) this.place_country = null;

            // optional location
            Object location_point_obj = lazyGet(json, "location_point");
            Object location_radius_obj = lazyGet(json, "location_radius");
            Object location_mark_obj = lazyGet(json, "location_mark");
            Object location_source_obj = lazyGet(json, "location_source");
            if (location_point_obj == null || location_mark_obj == null ||
                !(location_point_obj instanceof List<?>) ||
                !(location_mark_obj instanceof List<?>)) {
                this.location_point = null;
                this.location_radius = 0;
                this.location_mark = null;
                this.location_source = null;
            } else {
                this.location_point = new double[]{(Double) ((List<?>) location_point_obj).get(0), (Double) ((List<?>) location_point_obj).get(1)};
                this.location_radius = (int) MessageEntry.parseLong((Number) location_radius_obj);
                this.location_mark = new double[]{(Double) ((List<?>) location_mark_obj).get(0), (Double) ((List<?>) location_mark_obj).get(1)};
                this.location_source = LocationSource.valueOf((String) location_source_obj);
            }
            this.enriched = false;

            // load enriched data
            enrich();

            // may lead to error!!
            this.ready = new Semaphore(0);
            //this.user = null;
            //this.writeToIndex = false;
            //this.writeToBackend = false;
        }

        public TwitterTweet() throws MalformedURLException {
            this.moreData = new MessageEntry();
            this.timestamp = new Date().getTime();
            this.timestampDate = new Date(this.timestamp);
            this.created_at = new Date();
            this.on = null;
            this.to = null;
            this.source_type = SourceType.GENERIC;
            this.provider_type = ProviderType.NOONE;
            this.provider_hash = "";
            this.screen_name = "";
            this.retweet_from = "";
            this.postId = "";
            this.conversationID = "";
            this.conversationUserIDs = new LinkedHashSet<>();
            this.canonical_id = "";
            this.parent = "";
            this.text = "";
            this.status_id_url = null;
            this.reply_count = 0;
            this.retweet_count = 0;
            this.favourite_count = 0;
            this.images = new HashSet<String>();
            this.audios = new HashSet<String>();
            this.videos = new HashSet<String>();
            this.place_id = "";
            this.place_name = "";
            this.place_context = null;
            this.place_country = null;
            this.location_point = null;
            this.location_radius = 0;
            this.location_mark = null;
            this.location_source = null;
            this.without_l_len = 0;
            this.without_lu_len = 0;
            this.without_luh_len = 0;
            this.hosts = new ArrayList<String>();
            this.links = new ArrayList<String>();
            this.mentions = new ArrayList<String>();
            this.hashtags = new ArrayList<String>();
            this.moreData.classifier = null;
            this.enriched = false;

            // may lead to error!!
            this.ready = new Semaphore(0);
            //this.user = null;
            //this.writeToIndex = false;
            //this.writeToBackend = false;
        }

        //TODO: fix the location issue and shift to MessageEntry class
        public void getLocation() {
            if (this.text == null) this.text = "";

            if ((this.location_point == null || this.location_point.length == 0) && DAO.geoNames != null) {
                GeoMark loc = null;
                if (place_name != null && this.place_name.length() > 0 &&
                    (this.location_source == null || this.location_source == LocationSource.ANNOTATION || this.location_source == LocationSource.PLACE)) {
                    loc = DAO.geoNames.analyse(this.place_name, null, 5, Integer.toString(this.text.hashCode()));
                    this.place_context = PlaceContext.FROM;
                    this.location_source = LocationSource.PLACE;
                }
                if (loc == null) {
                    loc = DAO.geoNames.analyse(this.text, this.hashtags.toArray(new String[0]), 5, Integer.toString(this.text.hashCode()));
                    this.place_context = PlaceContext.ABOUT;
                    this.location_source = LocationSource.ANNOTATION;
                }
                if (loc != null) {
                    if (this.place_name == null || this.place_name.length() == 0) this.place_name = loc.getNames().iterator().next();
                    this.location_radius = 0;
                    this.location_point = new double[]{loc.lon(), loc.lat()}; //[longitude, latitude]
                    this.location_mark = new double[]{loc.mlon(), loc.mlat()}; //[longitude, latitude]
                    this.place_country = loc.getISO3166cc();
                }
            }
        }

        /**
         * create enriched data, useful for analytics and ranking:
         * - identify all mentioned users, hashtags and links
         * - count message size without links
         * - count message size without links and without users
         */
        public void enrich() {
            if (this.enriched) return;
            this.moreData.classifier = Classifier.classify(this.text);
        
            StringBuilder text = new StringBuilder(this.text);
            this.links = this.moreData.extractLinks(text.toString());
            text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
            // Text's length without link
            this.without_l_len = text.length();

            this.hosts = this.moreData.extractHosts(links);

            this.videos = this.moreData.getLinksVideo(this.links, this.videos);
            this.images = this.moreData.getLinksImage(this.links, this.images);
            this.audios = this.moreData.getLinksAudio(this.links, this.audios);

            this.users = this.moreData.extractUsers(text.toString());
            text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
            // Text's length without link and users
            this.without_lu_len = text.length();

            this.mentions = new ArrayList<String>();
            for (int i = 0; i < this.users.size(); i++) {
                this.mentions.add(this.users.get(i).substring(1));
            }

            this.hashtags = this.moreData.extractHashtags(text.toString());
            text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
            // Text's length without link, users and hashtags
            this.without_luh_len = text.length();

            getLocation();
            this.enriched = true;
        }

        /**
         * Channels on which the Tweet will be published -
         *      all
         *      twitter
         *      twitter/mention/*username*
         *      twitter/user/*username*         (User who posted the Tweet)
         *      twitter/hashtag/*hashtag*
         *      twitter/country/*country code*
         *      twitter/text/*token*
         * @return Array of channels to publish message to
         */
        @Override
        protected String[] getStreamChannels() {
            ArrayList<String> channels = new ArrayList<>();

            for (String mention : this.mentions) {
                channels.add("twitter/mention/" + mention);
            }

            for (String hashtag : this.hashtags) {
                channels.add("twitter/hashtag/" + hashtag);
            }

            channels.add("twitter/user/" + this.getScreenName());
            if (this.place_country != null) {
                channels.add("twitter/country/" + this.place_country);
            }

            for (String token : Classifier.normalize(this.text)) {
                channels.add("twitter/text/" + token);
            }

            channels.add("all");
            channels.add("twitter");

            return channels.toArray(new String[channels.size()]);
        }

        @Override
        public void run() {
            //long start = System.currentTimeMillis();
            try {
                //DAO.log("TwitterTweet [" + this.postId + "] start");
                this.text = unshorten(this.text);
                this.user.setName(unshorten(this.user.getName()));
                //DAO.log("TwitterTweet [" + this.postId + "] unshorten after " + (System.currentTimeMillis() - start) + "ms");
                this.enrich();

                //DAO.log("TwitterTweet [" + this.postId + "] enrich    after " + (System.currentTimeMillis() - start) + "ms");
                if (this.writeToIndex) IncomingMessageBuffer.addScheduler(this, this.user, true, true);
                //DAO.log("TwitterTweet [" + this.postId + "] write     after " + (System.currentTimeMillis() - start) + "ms");
                if (this.writeToBackend) DAO.outgoingMessages.transmitMessage(this, this.user);
                //DAO.log("TwitterTweet [" + this.postId + "] transmit  after " + (System.currentTimeMillis() - start) + "ms");
            } catch (Throwable e) {
                DAO.severe(e);
            } finally {
                this.ready.release(1000);
            }
        }

        public boolean isReady() {
            if (this.ready == null) throw new RuntimeException("isReady() should not be called if postprocessing is not started");
            return this.ready.availablePermits() > 0;
        }

        public boolean waitReady(long millis) {
            if (this.ready == null) throw new RuntimeException("waitReady() should not be called if postprocessing is not started");
            if (this.ready.availablePermits() > 0) return true;
            try {
                return this.ready.tryAcquire(millis, TimeUnit.MILLISECONDS);
            } catch (InterruptedException e) {
                return false;
            }
        }

        public Post toJSON() {
                // very important to include calculated data here because that is written
                // into the index using the abstract index factory
                return toJSON(null, true, Integer.MAX_VALUE, "");
            }

        public Post toJSON(final UserEntry user, final boolean calculatedData, final int iflinkexceedslength, final String urlstub) {

            // tweet data
            this.put(AbstractObjectEntry.TIMESTAMP_FIELDNAME, AbstractObjectEntry.utcFormatter.print(getTimestampDate().getTime()));
            this.put(AbstractObjectEntry.CREATED_AT_FIELDNAME, AbstractObjectEntry.utcFormatter.print(getCreatedAt().getTime()));
            if (this.on != null) this.put("on", AbstractObjectEntry.utcFormatter.print(this.on.getTime()));
            if (this.to != null) this.put("to", AbstractObjectEntry.utcFormatter.print(this.to.getTime()));
            this.put("screen_name", this.screen_name);
            if (this.retweet_from != null && this.retweet_from.length() > 0) this.put("retweet_from", this.retweet_from);
            // the tweet; the cleanup is a helper function which cleans mistakes from the past in scraping
            MessageEntry.TextLinkMap tlm = this.moreData.getText(iflinkexceedslength, urlstub, this.text, this.getLinks(), this.getPostId());
            this.put("text", tlm);
            if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm()); // this is the primary key for retrieval in elasticsearch
            this.put("id_str", this.postId);
            this.put("conversation_id", this.conversationID);
            this.put("conversation_user", this.conversationUserIDs);
            if (this.canonical_id != null) this.put("canonical_id", this.canonical_id);
            if (this.parent != null) this.put("parent", this.parent);
            this.put("source_type", this.source_type.toString());
            this.put("provider_type", this.provider_type.name());
            if (this.provider_hash != null && this.provider_hash.length() > 0) this.put("provider_hash", this.provider_hash);
            this.put("reply_count", this.reply_count);
            this.put("retweet_count", this.retweet_count);
            // there is a slight inconsistency here in the plural naming but thats how it is noted in the twitter api
            this.put("favourites_count", this.favourite_count);
            this.put("place_name", this.place_name);
            this.put("place_id", this.place_id);

            // add statistic/calculated data
            if (calculatedData) {

                // text length
                this.put("text_length", this.text.length());

                // location data
                if (this.place_context != null) this.put("place_context", this.place_context.name());
                if (this.place_country != null && this.place_country.length() == 2) {
                    this.put("place_country", DAO.geoNames.getCountryName(this.place_country));
                    this.put("place_country_code", this.place_country);
                    this.put("place_country_center", DAO.geoNames.getCountryCenter(this.place_country));
                }

                // add optional location data. This is written even if calculatedData == false if
                // the source is from REPORT to prevent that it is lost
                if (this.location_point != null && this.location_point.length == 2
                        && this.location_mark != null && this.location_mark.length == 2) {
                    // reference for this format:
                    // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html#_lat_lon_as_array_5
                    this.put("location_point", this.location_point); // [longitude, latitude]
                    this.put("location_radius", this.location_radius);
                    this.put("location_mark", this.location_mark);
                    this.put("location_source", this.location_source.name());
                }

                // redundant data for enhanced navigation with aggregations
                this.put("hosts", this.hosts);
                this.put("hosts_count", this.hosts.size());
                this.put("links", this.links);
                this.put("links_count", this.links.size());
                this.put("unshorten", tlm.short2long);
                this.put("images", this.images);
                this.put("images_count", this.images.size());
                this.put("audio", this.audios);
                this.put("audio_count", this.audios.size());
                this.put("videos", this.videos);
                this.put("videos_count", this.videos.size());
                this.put("mentions", this.mentions);
                this.put("mentions_count", this.mentions.size());
                this.put("hashtags", this.hashtags);
                this.put("hashtags_count", this.hashtags.size());

                // experimental, for ranking
                this.put("without_l_len", this.without_l_len);
                this.put("without_lu_len", this.without_lu_len);
                this.put("without_luh_len", this.without_luh_len);

                 // text classifier
                if (this.moreData.classifier != null) {
                    for (Map.Entry<Context, Classification<String, Category>> c: this.moreData.classifier.entrySet()) {
                        assert c.getValue() != null;
                        // we don't store non-existing classifications
                        if (c.getValue().getCategory() == Classifier.Category.NONE) continue;
                        this.put("classifier_" + c.getKey().name(), c.getValue().getCategory());
                        this.put("classifier_" + c.getKey().name() + "_probability",
                                c.getValue().getProbability() == Float.POSITIVE_INFINITY
                                        ? Float.MAX_VALUE : c.getValue().getProbability());
                    }
                }
            } else {
                this.remove("text_length");
                this.remove("place_context");
                this.remove("place_country");
                this.remove("place_country_code");
                this.remove("place_country_center");
                this.remove("location_point");
                this.remove("location_radius");
                this.remove("location_mark");
                this.remove("location_source");
                this.remove("hosts");
                this.remove("hosts_count");
                this.remove("links");
                this.remove("links_count");
                this.remove("unshorten");
                this.remove("images");
                this.remove("images_count");
                this.remove("audio");
                this.remove("audio_count");
                this.remove("videos");
                this.remove("videos_count");
                this.remove("mentions");
                this.remove("mentions_count");
                this.remove("hashtags");
                this.remove("hashtags_count");
                this.remove("without_l_len");
                this.remove("without_lu_len");
                this.remove("without_luh_len");
                if (this.moreData.classifier != null) {
                    for (Map.Entry<Context, Classification<String, Category>> c: this.moreData.classifier.entrySet()) {
                        this.remove("classifier_" + c.getKey().name());
                        this.remove("classifier_" + c.getKey().name() + "_probability");
                    }
                }
            }

            // add user
            if (user != null) this.put("user", user.toJSON());
            return this;
        }

        public boolean willBeTimeConsuming() {
            return timeline_link_pattern.matcher(this.text).find();
        }

       public Object lazyGet(JSONObject json, String key) {
            try {
                Object o = json.get(key);
                return o;
            } catch (JSONException e) {
                return null;
            }
        }

        public UserEntry getUser() {
            return this.user;
        }

        public Date getTimestampDate() {
            return this.timestampDate == null ? new Date() : this.timestampDate;
        }

        public Date getCreatedAt() {
            return this.created_at == null ? new Date() : this.created_at;
        }

        public void setCreatedAt(Date created_at) {
            this.created_at = created_at;
        }

        public Date getOn() {
            return this.on;
        }

        public void setOn(Date on) {
            this.on = on;
        }

        public Date getTo() {
            return this.to;
        }

        public void setTo(Date to) {
            this.to = to;
        }

        public SourceType getSourceType() {
            return this.source_type;
        }

        public void setSourceType(SourceType source_type) {
            this.source_type = source_type;
        }

        public ProviderType getProviderType() {
            return provider_type;
        }

        public void setProviderType(ProviderType provider_type) {
            this.provider_type = provider_type;
        }

        public String getProviderHash() {
            return provider_hash;
        }

        public void setProviderHash(String provider_hash) {
            this.provider_hash = provider_hash;
        }

        public String getScreenName() {
            return screen_name;
        }

        public void setScreenName(String user_screen_name) {
            this.screen_name = user_screen_name;
        }

        public String getRetweetFrom() {
            return this.retweet_from;
        }

        public void setRetweetFrom(String retweet_from) {
            this.retweet_from = retweet_from;
        }

        public URL getStatusIdUrl() {
            return this.status_id_url;
        }

        public void setStatusIdUrl(URL status_id_url) {
            this.status_id_url = status_id_url;
        }

        public long getRetweetCount() {
            return retweet_count;
        }

        public void setRetweetCount(long retweet_count) {
            this.retweet_count = retweet_count;
        }

        public long getFavouritesCount() {
            return this.favourite_count;
        }

        public void setFavouritesCount(long favourites_count) {
            this.favourite_count = favourites_count;
        }

        public String getPlaceName() {
            return place_name;
        }

        public void setPlaceName(String place_name, PlaceContext place_context) {
            this.place_name = place_name;
            this.place_context = place_context;
        }

        public PlaceContext getPlaceContext () {
            return place_context;
        }

        public String getPlaceId() {
            return place_id;
        }

        public void setPlaceId(String place_id) {
            this.place_id = place_id;
        }

        /**
         * @return [longitude, latitude]
         */
        public double[] getLocationPoint() {
            return location_point;
        }

        /**
         * set the location
         * @param location_point in the form [longitude, latitude]
         */
        public void setLocationPoint(double[] location_point) {
            this.location_point = location_point;
        }

        public String getPostId() {
            return String.valueOf(this.postId);
        }

        //TODO: to implement this method
        private void setPostId() {
            this.postId = String.valueOf(this.timestamp) + String.valueOf(this.created_at.getTime());
        }

        /**
         * @return [longitude, latitude] which is inside of getLocationRadius() from getLocationPoint()
         */
        public double[] getLocationMark() {
            return location_mark;
        }

        /**
         * Set the location
         * @param location_point in the form [longitude, latitude]
         */
        public void setLocationMark(double[] location_mark) {
            this.location_mark = location_mark;
        }

        /**
         * Get the radius in meter
         * @return radius in meter around getLocationPoint() (NOT getLocationMark())
         */
        public int getLocationRadius() {
            return location_radius;
        }

        public void setLocationRadius(int location_radius) {
            this.location_radius = location_radius;
        }

        public LocationSource getLocationSource() {
            return location_source;
        }

        public void setLocationSource(LocationSource location_source) {
            this.location_source = location_source;
        }

        public String getText() {
            return this.text;
        }

        public void setText(String text) {
            this.text = text;
        }

        public int getTextLength() {
            return this.text.length();
        }

        public long getId() {
            return Long.parseLong(this.postId);
        }

        public List<String> getHosts() {
            return this.hosts;
        }

        public Set<String> getVideos() {
            return this.videos;
        }

        public Set<String> getAudio() {
            return this.audios;
        }

        public Set<String> getImages() {
            return this.images;
        }

        public void setImages(String image) {
            if(this.images == null) {
                this.images = new HashSet<String>();
            }
            this.images.add(image);
        }

        public String[] getMentions() {
            if(this.mentions == null) {
                return new String[0];
            }
            return this.mentions.toArray(new String[0]);
        }

        public String[] getHashtags() {
            return this.hashtags.toArray(new String[0]);
        }

        public String[] getLinks() {
            return this.links.toArray(new String[0]);
        }

        public Classifier.Category getClassifier(Classifier.Context context) {
            return this.moreData.getClassifier(context);
        }

    }

    public static String unshorten(String text) {
        while (true) {
            try {
                Matcher m = emoji_pattern.matcher(text);
                if (m.find()) {
                    String emoji = m.group(1);
                    text = m.replaceFirst(emoji);
                    continue;
                }
            } catch (Throwable e) {
                DAO.severe(e);
                break;
            }
            try {
                Matcher m = emoji_pattern_span.matcher(text);
                if (m.find()) {
                    String emoji = m.group(1);
                    text = m.replaceFirst(emoji);
                    continue;
                }
            } catch (Throwable e) {
                DAO.severe(e);
                break;
            }
            try {
                Matcher m = hashtag_pattern.matcher(text);
                if (m.find()) {
                    text = m.replaceFirst(" #" + m.group(1) + " "); // the extra spaces are needed because twitter removes them if the hashtag is followed with a link
                    continue;
                }
            } catch (Throwable e) {
                DAO.severe(e);
                break;
            }
            try {
                Matcher m = timeline_link_pattern.matcher(text);
                if (m.find()) {
                    String expanded = RedirectUnshortener.unShorten(m.group(2));
                    text = m.replaceFirst(" " + expanded);
                    continue;
                }
            } catch (Throwable e) {
                DAO.severe(e);
                break;
            }
            try {
                Matcher m = timeline_embed_pattern.matcher(text);
                if (m.find()) {
                    text = m.replaceFirst("");
                    continue;
                }
            } catch (Throwable e) {
                DAO.severe(e);
                break;
            }
            break;
        }
        text = cleanup_pattern.matcher(text).replaceAll("");
        text = MessageEntry.html2utf8(text);
        text = doublespace_pattern.matcher(text).replaceAll(" ");
        text = text.trim();
        return text;
    }

    /**
     * Usage: java twitter4j.examples.search.SearchTweets [query]
     *
     * @param args search query
     */
    public static void main(String[] args) {
        //wget --no-check-certificate "https://twitter.com/search?q=eifel&src=typd&f=realtime"
        Set<String> filterList = new HashSet<String>();
        filterList.add("image");
        TwitterTimeline[] result = null;
        result = search(args[0], filterList, Order.CREATED_AT, false, false);
        int all = 0;
        for (int x = 0; x < 2; x++) {
            if (x == 0) System.out.println("Timeline[0] - finished to be used:");
            if (x == 1) System.out.println("Timeline[1] - messages which are in postprocessing");
            all += result[x].size();
            for (TwitterTweet tweet : result[x]) {
                    tweet.waitReady(10000);
                System.out.println(tweet.getCreatedAt().toString() + " from @" + tweet.getScreenName() + " - " + tweet.getText());
            }
        }
        System.out.println("count: " + all);
        System.exit(0);
    }

} public static class TwitterTweet extends Post implements Runnable {

    public final Semaphore ready;
    public MessageEntry moreData = new MessageEntry();
    public UserEntry user;
    public boolean writeToIndex;
    public boolean writeToBackend;

    // a time stamp that is given in loklak upon the arrival of the tweet which is the current local time
    public Date timestampDate;
    // the time given in the tweet which is the time when the user created it.
    // This is also use to do the index partition into minute, hour, week
    public Date created_at;
    // on means 'valid from'
    public Date on;
    // 'to' means 'valid_until' and may not be set
    public Date to;

    // where did the message come from
    protected SourceType source_type;
    // who created the message
    protected ProviderType provider_type;

    public String provider_hash, screen_name, retweet_from, postId, conversationID, canonical_id, parent, text;
    public Set<String> conversationUserIDs;
    protected URL status_id_url;
    protected long reply_count, retweet_count, favourite_count;
    public Set<String> images, audios, videos;
    protected String place_name, place_id;

    // the following fields are either set as a common field or generated by extraction from field 'text' or from field 'place_name'
    // coordinate order is [longitude, latitude]
    protected double[] location_point, location_mark;
    // Value in metres
    protected int location_radius;
    protected LocationSource location_source;
    protected PlaceContext place_context;
    protected String place_country;

    // The length of tweets without links, users, hashtags
    // the following can be computed from the tweet data but is stored in the search index
    // to provide statistical data and ranking attributes
    private int without_l_len, without_lu_len, without_luh_len;

    // the arrays of links, users, hashtags
    private List<String> users, hosts, links, mentions, hashtags;

    private boolean enriched;

    public TwitterTweet(
            final String user_screen_name_raw,
            final long created_at_raw,
            final String tweetID, // the ID of the tweet
            final String tweetConversationID, // the ID of a tweet where this is a reply-to tweet
            final Set<String> mentions_screennames,
            final String status_id_url_raw,
            final String text_raw,
            final long replies,
            final long retweets,
            final long favourites,
            final Set<String> images,
            final Set<String> videos,
            final String place_name,
            final String place_id,
            final UserEntry user,
            final boolean writeToIndex,
            final boolean writeToBackend) throws MalformedURLException {
        super();
        this.source_type = SourceType.TWITTER;
        this.provider_type = ProviderType.SCRAPED;
        this.screen_name = user_screen_name_raw;
        this.created_at = new Date(created_at_raw);
        this.status_id_url = new URL("https://twitter.com" + status_id_url_raw);
        int p = status_id_url_raw.lastIndexOf('/');
        this.postId = p >= 0 ? status_id_url_raw.substring(p + 1) : "-1";
        assert this.postId.equals(tweetID);
        this.conversationID = tweetConversationID;
        this.conversationUserIDs = mentions_screennames;
        this.reply_count = replies;
        this.retweet_count = retweets;
        this.favourite_count = favourites;
        this.place_name = place_name;
        this.place_id = place_id;
        this.images = images;
        this.videos = videos;
        this.text = text_raw;
        this.user = user;
        this.writeToIndex = writeToIndex;
        this.writeToBackend = writeToBackend;

        //Date d = new Date(timemsraw);
        //System.out.println(d);

        /* failed to reverse-engineering the place_id :(
        if (place_id.length() == 16) {
            String a = place_id.substring(0, 8);
            String b = place_id.substring(8, 16);
            long an = Long.parseLong(a, 16);
            long bn = Long.parseLong(b, 16);
            System.out.println("place = " + place_name + ", a = " + an + ", b = " + bn);
            // Frankfurt a = 3314145750, b = 3979907708, http://www.openstreetmap.org/#map=15/50.1128/8.6835
            // Singapore a = 1487192992, b = 3578663936
        }
        */

        // this.text MUST be analysed with analyse(); this is not done here because it should be started concurrently; run run();

        this.ready = new Semaphore(0);
    }

    public TwitterTweet(JSONObject json) {
        this.moreData = new MessageEntry();
        Object timestamp_obj = lazyGet(json, AbstractObjectEntry.TIMESTAMP_FIELDNAME);
        this.timestampDate = MessageEntry.parseDate(timestamp_obj);
        this.timestamp = this.timestampDate.getTime();
        Object created_at_obj = lazyGet(json, AbstractObjectEntry.CREATED_AT_FIELDNAME);
        this.created_at = MessageEntry.parseDate(created_at_obj);
        Object on_obj = lazyGet(json, "on");
        this.on = on_obj == null ? null : MessageEntry.parseDate(on);
        Object to_obj = lazyGet(json, "to");
        this.to = to_obj == null ? null : MessageEntry.parseDate(to);
        String source_type_string = (String) lazyGet(json, "source_type");
        try {
            this.source_type = source_type_string == null ? SourceType.GENERIC : SourceType.byName(source_type_string);
        } catch (IllegalArgumentException e) {
            this.source_type = SourceType.GENERIC;
        }
        String provider_type_string = (String) lazyGet(json, "provider_type");
        if (provider_type_string == null) provider_type_string = ProviderType.NOONE.name();
        try {
            this.provider_type = ProviderType.valueOf(provider_type_string);
        } catch (IllegalArgumentException e) {
            this.provider_type = ProviderType.NOONE;
        }
        this.provider_hash = (String) lazyGet(json, "provider_hash");
        this.screen_name = (String) lazyGet(json, "screen_name");
        this.retweet_from = (String) lazyGet(json, "retweet_from");
        this.postId = (String) lazyGet(json, "id_str");
        this.conversationID = (String) lazyGet(json, "conversationid_str");
        this.conversationUserIDs = MessageEntry.parseArrayList(lazyGet(json, "conversation_user"));
        this.text = (String) lazyGet(json, "text");
        try {
            this.status_id_url = new URL((String) lazyGet(json, "link"));
        } catch (MalformedURLException e) {
            this.status_id_url = null;
        }
        this.reply_count = MessageEntry.parseLong((Number) lazyGet(json, "reply_count"));
        this.retweet_count = MessageEntry.parseLong((Number) lazyGet(json, "retweet_count"));
        this.favourite_count = MessageEntry.parseLong((Number) lazyGet(json, "favourites_count")); // inconsitency in naming, but twitter api defines so
        this.images = MessageEntry.parseArrayList(lazyGet(json, "images"));
        this.audios = MessageEntry.parseArrayList(lazyGet(json, "audio"));
        this.videos = MessageEntry.parseArrayList(lazyGet(json, "videos"));
        this.place_id = MessageEntry.parseString((String) lazyGet(json, "place_id"));
        this.place_name = MessageEntry.parseString((String) lazyGet(json, "place_name"));
        this.place_country = MessageEntry.parseString((String) lazyGet(json, "place_country"));

        if (this.place_country != null && this.place_country.length() != 2) this.place_country = null;

        // optional location
        Object location_point_obj = lazyGet(json, "location_point");
        Object location_radius_obj = lazyGet(json, "location_radius");
        Object location_mark_obj = lazyGet(json, "location_mark");
        Object location_source_obj = lazyGet(json, "location_source");
        if (location_point_obj == null || location_mark_obj == null ||
            !(location_point_obj instanceof List<?>) ||
            !(location_mark_obj instanceof List<?>)) {
            this.location_point = null;
            this.location_radius = 0;
            this.location_mark = null;
            this.location_source = null;
        } else {
            this.location_point = new double[]{(Double) ((List<?>) location_point_obj).get(0), (Double) ((List<?>) location_point_obj).get(1)};
            this.location_radius = (int) MessageEntry.parseLong((Number) location_radius_obj);
            this.location_mark = new double[]{(Double) ((List<?>) location_mark_obj).get(0), (Double) ((List<?>) location_mark_obj).get(1)};
            this.location_source = LocationSource.valueOf((String) location_source_obj);
        }
        this.enriched = false;

        // load enriched data
        enrich();

        // may lead to error!!
        this.ready = new Semaphore(0);
        //this.user = null;
        //this.writeToIndex = false;
        //this.writeToBackend = false;
    }

    public TwitterTweet() throws MalformedURLException {
        this.moreData = new MessageEntry();
        this.timestamp = new Date().getTime();
        this.timestampDate = new Date(this.timestamp);
        this.created_at = new Date();
        this.on = null;
        this.to = null;
        this.source_type = SourceType.GENERIC;
        this.provider_type = ProviderType.NOONE;
        this.provider_hash = "";
        this.screen_name = "";
        this.retweet_from = "";
        this.postId = "";
        this.conversationID = "";
        this.conversationUserIDs = new LinkedHashSet<>();
        this.canonical_id = "";
        this.parent = "";
        this.text = "";
        this.status_id_url = null;
        this.reply_count = 0;
        this.retweet_count = 0;
        this.favourite_count = 0;
        this.images = new HashSet<String>();
        this.audios = new HashSet<String>();
        this.videos = new HashSet<String>();
        this.place_id = "";
        this.place_name = "";
        this.place_context = null;
        this.place_country = null;
        this.location_point = null;
        this.location_radius = 0;
        this.location_mark = null;
        this.location_source = null;
        this.without_l_len = 0;
        this.without_lu_len = 0;
        this.without_luh_len = 0;
        this.hosts = new ArrayList<String>();
        this.links = new ArrayList<String>();
        this.mentions = new ArrayList<String>();
        this.hashtags = new ArrayList<String>();
        this.moreData.classifier = null;
        this.enriched = false;

        // may lead to error!!
        this.ready = new Semaphore(0);
        //this.user = null;
        //this.writeToIndex = false;
        //this.writeToBackend = false;
    }

    //TODO: fix the location issue and shift to MessageEntry class
    public void getLocation() {
        if (this.text == null) this.text = "";

        if ((this.location_point == null || this.location_point.length == 0) && DAO.geoNames != null) {
            GeoMark loc = null;
            if (place_name != null && this.place_name.length() > 0 &&
                (this.location_source == null || this.location_source == LocationSource.ANNOTATION || this.location_source == LocationSource.PLACE)) {
                loc = DAO.geoNames.analyse(this.place_name, null, 5, Integer.toString(this.text.hashCode()));
                this.place_context = PlaceContext.FROM;
                this.location_source = LocationSource.PLACE;
            }
            if (loc == null) {
                loc = DAO.geoNames.analyse(this.text, this.hashtags.toArray(new String[0]), 5, Integer.toString(this.text.hashCode()));
                this.place_context = PlaceContext.ABOUT;
                this.location_source = LocationSource.ANNOTATION;
            }
            if (loc != null) {
                if (this.place_name == null || this.place_name.length() == 0) this.place_name = loc.getNames().iterator().next();
                this.location_radius = 0;
                this.location_point = new double[]{loc.lon(), loc.lat()}; //[longitude, latitude]
                this.location_mark = new double[]{loc.mlon(), loc.mlat()}; //[longitude, latitude]
                this.place_country = loc.getISO3166cc();
            }
        }
    }

    /**
     * create enriched data, useful for analytics and ranking:
     * - identify all mentioned users, hashtags and links
     * - count message size without links
     * - count message size without links and without users
     */
    public void enrich() {
        if (this.enriched) return;
        this.moreData.classifier = Classifier.classify(this.text);
        
        StringBuilder text = new StringBuilder(this.text);
        this.links = this.moreData.extractLinks(text.toString());
        text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
        // Text's length without link
        this.without_l_len = text.length();

        this.hosts = this.moreData.extractHosts(links);

        this.videos = this.moreData.getLinksVideo(this.links, this.videos);
        this.images = this.moreData.getLinksImage(this.links, this.images);
        this.audios = this.moreData.getLinksAudio(this.links, this.audios);

        this.users = this.moreData.extractUsers(text.toString());
        text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
        // Text's length without link and users
        this.without_lu_len = text.length();

        this.mentions = new ArrayList<String>();
        for (int i = 0; i < this.users.size(); i++) {
            this.mentions.add(this.users.get(i).substring(1));
        }

        this.hashtags = this.moreData.extractHashtags(text.toString());
        text = new StringBuilder(MessageEntry.SPACEX_PATTERN.matcher(text).replaceAll(" ").trim());
        // Text's length without link, users and hashtags
        this.without_luh_len = text.length();

        getLocation();
        this.enriched = true;
    }

    /**
     * Channels on which the Tweet will be published -
     *      all
     *      twitter
     *      twitter/mention/*username*
     *      twitter/user/*username*         (User who posted the Tweet)
     *      twitter/hashtag/*hashtag*
     *      twitter/country/*country code*
     *      twitter/text/*token*
     * @return Array of channels to publish message to
     */
    @Override
    protected String[] getStreamChannels() {
        ArrayList<String> channels = new ArrayList<>();

        for (String mention : this.mentions) {
            channels.add("twitter/mention/" + mention);
        }

        for (String hashtag : this.hashtags) {
            channels.add("twitter/hashtag/" + hashtag);
        }

        channels.add("twitter/user/" + this.getScreenName());
        if (this.place_country != null) {
            channels.add("twitter/country/" + this.place_country);
        }

        for (String token : Classifier.normalize(this.text)) {
            channels.add("twitter/text/" + token);
        }

        channels.add("all");
        channels.add("twitter");

        return channels.toArray(new String[channels.size()]);
    }

    @Override
    public void run() {
        //long start = System.currentTimeMillis();
        try {
            //DAO.log("TwitterTweet [" + this.postId + "] start");
            this.text = unshorten(this.text);
            this.user.setName(unshorten(this.user.getName()));
            //DAO.log("TwitterTweet [" + this.postId + "] unshorten after " + (System.currentTimeMillis() - start) + "ms");
            this.enrich();

            //DAO.log("TwitterTweet [" + this.postId + "] enrich    after " + (System.currentTimeMillis() - start) + "ms");
            if (this.writeToIndex) IncomingMessageBuffer.addScheduler(this, this.user, true, true);
            //DAO.log("TwitterTweet [" + this.postId + "] write     after " + (System.currentTimeMillis() - start) + "ms");
            if (this.writeToBackend) DAO.outgoingMessages.transmitMessage(this, this.user);
            //DAO.log("TwitterTweet [" + this.postId + "] transmit  after " + (System.currentTimeMillis() - start) + "ms");
        } catch (Throwable e) {
            DAO.severe(e);
        } finally {
            this.ready.release(1000);
        }
    }

    public boolean isReady() {
        if (this.ready == null) throw new RuntimeException("isReady() should not be called if postprocessing is not started");
        return this.ready.availablePermits() > 0;
    }

    public boolean waitReady(long millis) {
        if (this.ready == null) throw new RuntimeException("waitReady() should not be called if postprocessing is not started");
        if (this.ready.availablePermits() > 0) return true;
        try {
            return this.ready.tryAcquire(millis, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            return false;
        }
    }

    public Post toJSON() {
            // very important to include calculated data here because that is written
            // into the index using the abstract index factory
            return toJSON(null, true, Integer.MAX_VALUE, "");
        }

    public Post toJSON(final UserEntry user, final boolean calculatedData, final int iflinkexceedslength, final String urlstub) {

        // tweet data
        this.put(AbstractObjectEntry.TIMESTAMP_FIELDNAME, AbstractObjectEntry.utcFormatter.print(getTimestampDate().getTime()));
        this.put(AbstractObjectEntry.CREATED_AT_FIELDNAME, AbstractObjectEntry.utcFormatter.print(getCreatedAt().getTime()));
        if (this.on != null) this.put("on", AbstractObjectEntry.utcFormatter.print(this.on.getTime()));
        if (this.to != null) this.put("to", AbstractObjectEntry.utcFormatter.print(this.to.getTime()));
        this.put("screen_name", this.screen_name);
        if (this.retweet_from != null && this.retweet_from.length() > 0) this.put("retweet_from", this.retweet_from);
        // the tweet; the cleanup is a helper function which cleans mistakes from the past in scraping
        MessageEntry.TextLinkMap tlm = this.moreData.getText(iflinkexceedslength, urlstub, this.text, this.getLinks(), this.getPostId());
        this.put("text", tlm);
        if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm()); // this is the primary key for retrieval in elasticsearch
        this.put("id_str", this.postId);
        this.put("conversation_id", this.conversationID);
        this.put("conversation_user", this.conversationUserIDs);
        if (this.canonical_id != null) this.put("canonical_id", this.canonical_id);
        if (this.parent != null) this.put("parent", this.parent);
        this.put("source_type", this.source_type.toString());
        this.put("provider_type", this.provider_type.name());
        if (this.provider_hash != null && this.provider_hash.length() > 0) this.put("provider_hash", this.provider_hash);
        this.put("reply_count", this.reply_count);
        this.put("retweet_count", this.retweet_count);
        // there is a slight inconsistency here in the plural naming but thats how it is noted in the twitter api
        this.put("favourites_count", this.favourite_count);
        this.put("place_name", this.place_name);
        this.put("place_id", this.place_id);

        // add statistic/calculated data
        if (calculatedData) {

            // text length
            this.put("text_length", this.text.length());

            // location data
            if (this.place_context != null) this.put("place_context", this.place_context.name());
            if (this.place_country != null && this.place_country.length() == 2) {
                this.put("place_country", DAO.geoNames.getCountryName(this.place_country));
                this.put("place_country_code", this.place_country);
                this.put("place_country_center", DAO.geoNames.getCountryCenter(this.place_country));
            }

            // add optional location data. This is written even if calculatedData == false if
            // the source is from REPORT to prevent that it is lost
            if (this.location_point != null && this.location_point.length == 2
                    && this.location_mark != null && this.location_mark.length == 2) {
                // reference for this format:
                // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html#_lat_lon_as_array_5
                this.put("location_point", this.location_point); // [longitude, latitude]
                this.put("location_radius", this.location_radius);
                this.put("location_mark", this.location_mark);
                this.put("location_source", this.location_source.name());
            }

            // redundant data for enhanced navigation with aggregations
            this.put("hosts", this.hosts);
            this.put("hosts_count", this.hosts.size());
            this.put("links", this.links);
            this.put("links_count", this.links.size());
            this.put("unshorten", tlm.short2long);
            this.put("images", this.images);
            this.put("images_count", this.images.size());
            this.put("audio", this.audios);
            this.put("audio_count", this.audios.size());
            this.put("videos", this.videos);
            this.put("videos_count", this.videos.size());
            this.put("mentions", this.mentions);
            this.put("mentions_count", this.mentions.size());
            this.put("hashtags", this.hashtags);
            this.put("hashtags_count", this.hashtags.size());

            // experimental, for ranking
            this.put("without_l_len", this.without_l_len);
            this.put("without_lu_len", this.without_lu_len);
            this.put("without_luh_len", this.without_luh_len);

             // text classifier
            if (this.moreData.classifier != null) {
                for (Map.Entry<Context, Classification<String, Category>> c: this.moreData.classifier.entrySet()) {
                    assert c.getValue() != null;
                    // we don't store non-existing classifications
                    if (c.getValue().getCategory() == Classifier.Category.NONE) continue;
                    this.put("classifier_" + c.getKey().name(), c.getValue().getCategory());
                    this.put("classifier_" + c.getKey().name() + "_probability",
                            c.getValue().getProbability() == Float.POSITIVE_INFINITY
                                    ? Float.MAX_VALUE : c.getValue().getProbability());
                }
            }
        } else {
            this.remove("text_length");
            this.remove("place_context");
            this.remove("place_country");
            this.remove("place_country_code");
            this.remove("place_country_center");
            this.remove("location_point");
            this.remove("location_radius");
            this.remove("location_mark");
            this.remove("location_source");
            this.remove("hosts");
            this.remove("hosts_count");
            this.remove("links");
            this.remove("links_count");
            this.remove("unshorten");
            this.remove("images");
            this.remove("images_count");
            this.remove("audio");
            this.remove("audio_count");
            this.remove("videos");
            this.remove("videos_count");
            this.remove("mentions");
            this.remove("mentions_count");
            this.remove("hashtags");
            this.remove("hashtags_count");
            this.remove("without_l_len");
            this.remove("without_lu_len");
            this.remove("without_luh_len");
            if (this.moreData.classifier != null) {
                for (Map.Entry<Context, Classification<String, Category>> c: this.moreData.classifier.entrySet()) {
                    this.remove("classifier_" + c.getKey().name());
                    this.remove("classifier_" + c.getKey().name() + "_probability");
                }
            }
        }

        // add user
        if (user != null) this.put("user", user.toJSON());
        return this;
    }

    public boolean willBeTimeConsuming() {
        return timeline_link_pattern.matcher(this.text).find();
    }

   public Object lazyGet(JSONObject json, String key) {
        try {
            Object o = json.get(key);
            return o;
        } catch (JSONException e) {
            return null;
        }
    }

    public UserEntry getUser() {
        return this.user;
    }

    public Date getTimestampDate() {
        return this.timestampDate == null ? new Date() : this.timestampDate;
    }

    public Date getCreatedAt() {
        return this.created_at == null ? new Date() : this.created_at;
    }

    public void setCreatedAt(Date created_at) {
        this.created_at = created_at;
    }

    public Date getOn() {
        return this.on;
    }

    public void setOn(Date on) {
        this.on = on;
    }

    public Date getTo() {
        return this.to;
    }

    public void setTo(Date to) {
        this.to = to;
    }

    public SourceType getSourceType() {
        return this.source_type;
    }

    public void setSourceType(SourceType source_type) {
        this.source_type = source_type;
    }

    public ProviderType getProviderType() {
        return provider_type;
    }

    public void setProviderType(ProviderType provider_type) {
        this.provider_type = provider_type;
    }

    public String getProviderHash() {
        return provider_hash;
    }

    public void setProviderHash(String provider_hash) {
        this.provider_hash = provider_hash;
    }

    public String getScreenName() {
        return screen_name;
    }

    public void setScreenName(String user_screen_name) {
        this.screen_name = user_screen_name;
    }

    public String getRetweetFrom() {
        return this.retweet_from;
    }

    public void setRetweetFrom(String retweet_from) {
        this.retweet_from = retweet_from;
    }

    public URL getStatusIdUrl() {
        return this.status_id_url;
    }

    public void setStatusIdUrl(URL status_id_url) {
        this.status_id_url = status_id_url;
    }

    public long getRetweetCount() {
        return retweet_count;
    }

    public void setRetweetCount(long retweet_count) {
        this.retweet_count = retweet_count;
    }

    public long getFavouritesCount() {
        return this.favourite_count;
    }

    public void setFavouritesCount(long favourites_count) {
        this.favourite_count = favourites_count;
    }

    public String getPlaceName() {
        return place_name;
    }

    public void setPlaceName(String place_name, PlaceContext place_context) {
        this.place_name = place_name;
        this.place_context = place_context;
    }

    public PlaceContext getPlaceContext () {
        return place_context;
    }

    public String getPlaceId() {
        return place_id;
    }

    public void setPlaceId(String place_id) {
        this.place_id = place_id;
    }

    /**
     * @return [longitude, latitude]
     */
    public double[] getLocationPoint() {
        return location_point;
    }

    /**
     * set the location
     * @param location_point in the form [longitude, latitude]
     */
    public void setLocationPoint(double[] location_point) {
        this.location_point = location_point;
    }

    public String getPostId() {
        return String.valueOf(this.postId);
    }

    //TODO: to implement this method
    private void setPostId() {
        this.postId = String.valueOf(this.timestamp) + String.valueOf(this.created_at.getTime());
    }

    /**
     * @return [longitude, latitude] which is inside of getLocationRadius() from getLocationPoint()
     */
    public double[] getLocationMark() {
        return location_mark;
    }

    /**
     * Set the location
     * @param location_point in the form [longitude, latitude]
     */
    public void setLocationMark(double[] location_mark) {
        this.location_mark = location_mark;
    }

    /**
     * Get the radius in meter
     * @return radius in meter around getLocationPoint() (NOT getLocationMark())
     */
    public int getLocationRadius() {
        return location_radius;
    }

    public void setLocationRadius(int location_radius) {
        this.location_radius = location_radius;
    }

    public LocationSource getLocationSource() {
        return location_source;
    }

    public void setLocationSource(LocationSource location_source) {
        this.location_source = location_source;
    }

    public String getText() {
        return this.text;
    }

    public void setText(String text) {
        this.text = text;
    }

    public int getTextLength() {
        return this.text.length();
    }

    public long getId() {
        return Long.parseLong(this.postId);
    }

    public List<String> getHosts() {
        return this.hosts;
    }

    public Set<String> getVideos() {
        return this.videos;
    }

    public Set<String> getAudio() {
        return this.audios;
    }

    public Set<String> getImages() {
        return this.images;
    }

    public void setImages(String image) {
        if(this.images == null) {
            this.images = new HashSet<String>();
        }
        this.images.add(image);
    }

    public String[] getMentions() {
        if(this.mentions == null) {
            return new String[0];
        }
        return this.mentions.toArray(new String[0]);
    }

    public String[] getHashtags() {
        return this.hashtags.toArray(new String[0]);
    }

    public String[] getLinks() {
        return this.links.toArray(new String[0]);
    }

    public Classifier.Category getClassifier(Classifier.Context context) {
        return this.moreData.getClassifier(context);
    }

} public class InstagramProfileScraper extends BaseScraper {

    private static final long serialVersionUID = -3360416757176406604L;
    
    /**
     * instaJsonData is a Pattern object which will be used to match the required pattern
     * with the extracted elements from html page
     */
    private Pattern instaJsonData = Pattern.compile("(\\{\"config\").*(\\});");

    /**
     * Constructor to set baseUrl and scraperName in the super(Base-Interface) for
     * current Search-Scrapper
     */
    public InstagramProfileScraper() {
        super();
        this.baseUrl = "https://www.instagram.com/";
        this.scraperName = "instagram";
    }

    /**
     * Constructor to set the given query and map _extra
     * @param _extra
     * A map with key and value as string
     * @param _query
     * _query in form for string value
     */
    public InstagramProfileScraper(String _query, Map<String, String> _extra) {
        this();
        this.setExtra(_extra);
        this.query = _query;
    }

    /**
     * Constructor to map the given _extra param with key and value as String
     */
    public InstagramProfileScraper(Map<String, String> _extra) {
        this();
        this.setExtra(_extra);
    }

    /**
     * Constructor to set String query
     * @param query as string
     */
    public InstagramProfileScraper(String _query) {
        this();
        this.query = _query;
    }

    /**
     * Method to get api path of the Instagram Profile Scraper
     * @return api endpoint of Instagram Profile Scraper in form of String
     */
	@Override
	public String getAPIPath() {
		return "/api/instagramprofilescraper.json";
	}

	@Override
	public BaseUserRole getMinimalBaseUserRole() {
		return BaseUserRole.ANONYMOUS;
	}

    /**
     * @return null when asked for the default permissions
     * of the given base user role in form of JSONObject
     */
	@Override
	public JSONObject getDefaultPermissions(BaseUserRole baseUserRole) {
		return null;
	}

    /**
     * Method to generate url for search using URIBuilder
     * @return URL in string format
     */
    protected String prepareSearchUrl(String type) {
        URIBuilder url = null;
        try {
            url = new URIBuilder(this.baseUrl + this.query);
        } catch (URISyntaxException e) {
            DAO.log("Invalid Url: baseUrl = " + this.baseUrl + ", mid-URL = " + midUrl + "query = " + this.query + "type = " + type);
            return "";
        }

        return url.toString();
    }

    /**
     * Method to set the "profile" key with value of query if the "profile" key is empty/null
     * and set the "query" key with value of query
     */
    protected void setParam() {
        if("".equals(this.getExtraValue("profile"))) {
            this.setExtraValue("profile", this.query);
        }
        this.setExtraValue("query", this.query);
    }

    /**
     * Method to scrape the given url with key as "profile_posts"
     * @return instaObj as a Post object 
     */
    protected Post scrape(BufferedReader br, String type, String url) {
   	    Post instaObj = new Post(true);
        this.putData(instaObj, "profile_posts", this.scrapeInstagram(br, url));
        return instaObj;
    }

    /**
     * Method to match the given pattern with extracted elements of html page
     * and parse the result for the posts on the given instagram page
     * @return instaProfile as a JSONArray object containing all posts and details of viewer
     */
	public JSONArray scrapeInstagram(BufferedReader br, String url) {
		Document htmlPage = null;
        Post instaObj = null;
        JSONArray instaProfile = new JSONArray();
        try {
            htmlPage = Jsoup.parse(this.bufferedReaderToString(br));
        } catch (IOException e) {
            DAO.trace(e);
		}

		String script = htmlPage.getElementsByTag("script").html();
        Matcher m = instaJsonData.matcher(script);
        m.find();
        int start = m.start(1);
        int end = m.start(2) + 1;
        script = script.substring(start, end);

        //TODO: pre-process the posts captured. At present, complete array of posts are output.
        //Only useful data shall be outputted.
		instaObj = new Post(script, this.query);
        instaProfile.put(instaObj);
       return instaProfile;
	}

} public class QuoraProfileScraper extends BaseScraper {

    private static final long serialVersionUID = -3398701925784347312L;
    private List<String> typeList = null;

    public QuoraProfileScraper() {
        super();
        this.baseUrl = "https://www.quora.com/";
        this.scraperName = "quora";
        this.sourceType = SourceType.QUORA;
    }

    public QuoraProfileScraper(String _query) {
        this();
        this.query = _query;
        this.setExtraValue("query", this.query);
    }

    public QuoraProfileScraper(String _query, Map<String, String> _extra) {
        this();
        this.setExtra(_extra);
        this.query = _query;
        this.setExtraValue("query", this.query);
    }

    public QuoraProfileScraper(Map<String, String> _extra) {
        this();
        this.setExtra(_extra);
    }

    protected void setParam() {
        if(!"".equals(this.getExtraValue("type"))) {
            this.typeList = Arrays.asList(this.getExtraValue("type").trim().split("\\s*,\\s*"));
        } else {
            this.typeList = new ArrayList<String>();
            this.typeList.add("all");
            this.setExtraValue("type", String.join(",", this.typeList));
        }
        this.query = this.getExtraValue("query");
    }

    @Override
    protected void setCacheMap() {
        this.cacheMap = new HashMap<String, Map<String, String>>();

        Map<String, String> getMap = new HashMap<String, String>();
        if(this.typeList.contains("user") && this.typeList.size() == 1) {
            getMap.put("post_type", "user");
            getMap.put("user_name", this.query);
        } else if(this.typeList.contains("question") && this.typeList.size() == 1) {
            getMap.put("post_type", "question");
            getMap.put("post_ques", this.query);
        }
        getMap.put("post_scraper", this.scraperName);

        Map<String, String> alsoGetMap = new HashMap<String, String>();
        if(this.typeList.contains("question") || this.typeList.contains("all")) {
            alsoGetMap.put("post_ques", this.query);
        }
        if(this.typeList.contains("user") || this.typeList.contains("all")) {
            alsoGetMap.put("user_name", this.query);
        }

        this.cacheMap.put("get", getMap);
        this.cacheMap.put("also_get", alsoGetMap);
    }

    @Override
    public String getAPIPath() {
        return "/api/quoraprofilescraper.json";
    }

    @Override
    public BaseUserRole getMinimalBaseUserRole() {
        return BaseUserRole.ANONYMOUS;
    }

    @Override
    public JSONObject getDefaultPermissions(BaseUserRole baseUserRole) {
        return null;
    }

    protected Map<?, ?> getExtra(String _extra) {
        return new HashMap<String, String>();
    }

    protected String prepareSearchUrl(String type) {
        URIBuilder url = null;
        String midUrl = "search/";
        try {
            switch(type) {
                case "user":
                    midUrl = "profile/";
                    url = new URIBuilder(this.baseUrl + midUrl + this.query);
                    break;
                case "question":
                    url = new URIBuilder(this.baseUrl + midUrl);
                    url.addParameter("q", this.query);
                    url.addParameter("type", "question");
                    break;
                case "answer":
                    url = new URIBuilder(this.baseUrl + midUrl);
                    url.addParameter("q", this.query);
                    url.addParameter("type", "answer");
                    break;
                default:
                    url = new URIBuilder("");
                    break;
            }
        } catch (URISyntaxException e) {
            DAO.log("Invalid Url: baseUrl = " + this.baseUrl + ", mid-URL = " + midUrl + "query = " + this.query + "type = " + type);
        }

        return url.toString();
    }

    @Override
    public Post getResults() {
        String url;
        Thread[] dataThreads = new Thread[2];
        PostTimeline postList = new PostTimeline(this.order);

        if(this.typeList.contains("user") || this.typeList.contains("all")) {
            url = prepareSearchUrl("user");
            dataThreads[0] = new ConcurrentScrape(url, "users", postList);
            dataThreads[0].start();
        } else {
            dataThreads[0] = new Thread();
        }
        if(this.typeList.contains("question") || this.typeList.contains("all")) {
            url = prepareSearchUrl("question");
            dataThreads[1] = new ConcurrentScrape(url, "question", postList);
            dataThreads[1].start();
        } else {
            dataThreads[1] = new Thread();
        }
        //TODO: add more types

        int i = 0;
        try {
            for (i = 0; i < dataThreads.length; i++) {
                dataThreads[i].join();
            }
        } catch(InterruptedException e) {
            exceptionOutputGetData(i);
        }

        // Add scraper name
        Post postArray = new Post();
        postArray.put(this.scraperName, postList.toArray());

        return postArray;
    }

    protected class ConcurrentScrape extends Thread {

        private String url = "";
        private String type = "all";
        private PostTimeline postList = null;

        public ConcurrentScrape(String url, String type, PostTimeline postList) {
            this.url = url;
            this.type = type;
            this.postList = postList;
        }

        public void run() {
            try {
                this.postList.addPost(QuoraProfileScraper.this.getDataFromConnection(this.url, this.type));
            } catch (IOException e) {
                DAO.severe("check internet connection for url: " + this.url + " type: " + this.type);
            }
        }
    }

    private void exceptionOutputGetData(int i) {
        String stuck_at = "";
        switch(i) {
            case 0:
                stuck_at = "users";
                break;
            case 1:
                stuck_at = "question";
                break;
            case 2:
                stuck_at = "answer";
                break;
            default:
                stuck_at = "unknown, check the code";
        }
        DAO.severe("Couldn't complete all threads, stuck at scraper: " + this.scraperName + " dataThread: " + stuck_at);
    }

    public PostTimeline scrapeQues(BufferedReader br, String url) {
        Pattern resultBlock = Pattern.compile("<div[^>]*[^>\\s]*[^>]*class=['\"][^>'\"]*(results_list)");
        Pattern quesLink = Pattern.compile("<a[^>]*class=['\"][^>'\"]*question_link[^>'\"]*[\"'][^>]*href=['\"]([^>'\"]*)");
        Pattern quesStart = Pattern.compile("<span[^>]*class=[\'\"][^>\'\"]*question_text[^>\'\"]*[\'\"][^>]*>");
        String ignoreTag = "(<[^>]*>)";
        Matcher matcher;
        String fromTerm = "";
        String uptoTerm = "</a>";
        Post qPost = null;

        PostTimeline quesList = new PostTimeline(this.order);

        String input = "";
        String _qPostId = "";
        String _qPostUrl = "";

        int last = 0;
        try {
            // Get to Result List block
            while(true) {
                input = br.readLine();
                if (input == null) break;

                matcher = resultBlock.matcher(input);
                if (matcher.find()) {
                    input = input.substring(matcher.end());
                    break;
                }
            }
            // Scraping starts
            for(int i = 0;input !=null ; i++) {

                // Get to first result
                while(input != null) {
                    matcher = quesLink.matcher(input);
                    if(matcher.find()) {
                        _qPostId = matcher.group(1).substring(1);
                        _qPostUrl = this.baseUrl + _qPostId;
                        input = input.substring(matcher.end());
                        break;
                    } else {
                        input = br.readLine();
                    }

                }

                qPost = new QuoraPost(_qPostId, i);
                qPost.put("search_url", url);
                qPost.put("post_url", _qPostUrl);
                qPost.put("post_type", "question");
                qPost.put("post_scraper", "quora");
                // Get questions
                while(input != null) {
                    matcher = quesStart.matcher(input);
                    if(matcher.find()) {
                        input = input.substring(matcher.end());
                        break;
                    } else {
                        input = br.readLine();
                    }
                }

                while(input != null) {
                    fromTerm = fromTerm + input;
                    last = input.indexOf(uptoTerm);
                    if(input.indexOf(uptoTerm) > 0) {
                        fromTerm = fromTerm.substring(0, input.indexOf(uptoTerm));
                        break;
                    } else {
                        input = br.readLine();
                    }

                }

                fromTerm = fromTerm.replaceAll(ignoreTag, "");
                input = input.substring(last);
                qPost.put("post_ques", fromTerm);
                fromTerm = "";
                quesList.addPost(qPost);
            }


        } catch(IOException e) {
            qPost = new QuoraPost(this.query, -1);
            qPost.put("error", "Connection error while fetching");
            qPost.put("search_url", url);
            quesList.addPost(qPost);
        } catch(NullPointerException e) { }

        return quesList;
    }

    public PostTimeline scrapeProfile(BufferedReader br, String url) {
        String html;
        Post quoraProfile = new QuoraPost(this.query, 0);
        PostTimeline usersList = new PostTimeline(this.order);
        try {
            html = bufferedReaderToString(br);
        } catch(IOException e) {
            DAO.trace(e);
            html = "";
            //TODO: output error if no output in json
        }

        Document userHTML = Jsoup.parse(html);

        quoraProfile.put("search_url", url);
        quoraProfile.put("post_type", "user");
        quoraProfile.put("post_scraper", this.scraperName);
        String bio = userHTML.getElementsByAttributeValueContaining("class", "ProfileDescription").text();
        quoraProfile.put("bio", bio);

        String profileImage = userHTML.getElementsByAttributeValueContaining("class", "profile_photo_img").attr("src");
        quoraProfile.put("profileImage", profileImage);

        String userName = userHTML.getElementsByAttributeValueContaining("class", "profile_photo_img").attr("alt");
        quoraProfile.put("user_name", userName);

        String rssFeedLink = url + "/rss";
        quoraProfile.put("rss_feed_link", rssFeedLink);

        Elements userBasicInfo = userHTML.getElementsByAttributeValueContaining("class", "UserCredential IdentityCredential");
        for (Element info: userBasicInfo) {
            String infoText = info.text();
            if (infoText.startsWith("Studi")) {
                quoraProfile.put(infoText.split(" ")[0].toLowerCase().trim() + "_at", infoText);
            } else if (infoText.startsWith("Lives")) {
                quoraProfile.put("lives_in", infoText);
            } else {
                quoraProfile.put("works_at", infoText);
            }
        }

        Elements knowsAbout = userHTML.getElementsByAttributeValueContaining("class",  "TopicNameSpan TopicName");
        JSONArray topics = new JSONArray();
        for (Element topic: knowsAbout) {
            topics.put(topic.text());
        }
        quoraProfile.put("knows_about", topics);

        JSONObject feeds = new JSONObject();
        Elements counts = userHTML.getElementsByAttributeValueContaining("class", "list_count");
        for (Element count: counts) {
            String topic = count.parent().text();
            topic = topic.substring(0, topic.indexOf(count.text())).trim();
            feeds.put(topic.toLowerCase() + "_url", baseUrl + count.parent().attr("href").substring(1));
            feeds.put(topic.toLowerCase(), count.text());
        }
        quoraProfile.put("feeds", feeds);

        usersList.addPost(quoraProfile);
        return usersList;
    }

    @Override
    protected Post scrape(BufferedReader br, String type, String url) {
        Post typeArray = new Post(true);

        switch(type) {
            case "users":
                this.putData(typeArray, "users", this.scrapeProfile(br, url));
                break;
            case "question":
                this.putData(typeArray, "question", this.scrapeQues(br, url));
                break;
            default:
                break;
        }
        return typeArray;
    }

    public static class QuoraPost extends Post {

        //quora post-id, for profile it will be username
        private String quoraId;
        private int quoraPostNo;

        public QuoraPost(String _quoraId, int _quoraPostNo) {
            //not UTC, may be error prone
            super();
            this.quoraPostNo = _quoraPostNo;
            this.setPostId(_quoraId);
        }

        public void getQuoraId(String _quoraId) {
            this.quoraId = _quoraId;
        }

        public void getQuoraPostNo(int _quoraPostNo) {
            this.quoraPostNo = _quoraPostNo;
        }

        public String getPostId() {
            return String.valueOf(this.postId);
        }
        //clean data
    }
} public abstract class BaseScraper extends AbstractAPIHandler {

    /**
     * 
     */
    private static final long serialVersionUID = 2088186173633845169L;
    // a time stamp that is given by loklak upon the arrival of the post which is the current local time
    //TODO: check if UTC time needed
    protected String scraperName;
    protected String html;
    protected String baseUrl;
    protected String midUrl;
    protected String query = null;
    protected String source;
    protected int resultCount = 0;
    // where did the message come from
    protected SourceType sourceType = SourceType.GENERIC;
    // who created the message
    protected ProviderType provider_type;
    protected Map<String, String> extra = null;
    protected Map<String, Map<String, String>> cacheMap = null;
    protected final PostTimeline.Order order = PostTimeline.parseOrder("timestamp");
    protected int hits = 0;
    protected int count = 0;

    @Override
    public JSONObject serviceImpl(Query call, HttpServletResponse response, Authorization rights,
            JSONObjectWithDefault permissions) throws APIException {
        this.setExtra(call);
        this.setParam();
        return this.getData();
    }

    protected void setExtra(Query call) {
        this.setExtra(call.getMap());
    }

    protected void setExtra(Map<String, String> _extra) {
        this.extra = _extra;
        this.query = _extra.get("query");
        try{
            this.resultCount = Integer.parseInt(_extra.get("count"));
        } catch (NumberFormatException e) {
            //TODO: set with config file
            this.resultCount = 10;
        }
        this.source = _extra.get("source");
        this.setParam();
    }

    public String getExtraValue(String key) {
        String value = "";
        if(this.extra.get(key) != null) {
            value = this.extra.get(key).trim();
        }
        return value;
    }

    protected void setExtraValue(String key, String value) {
        if(this.extra == null) {
            this.extra = new HashMap<String, String>();
        }
        this.extra.put(key, value);
    }

    protected void setCacheMap() {
        cacheMap = null;
    }

    protected abstract void setParam();

    protected abstract String prepareSearchUrl(String type);

    public Post getData() {
        Post output = new Post(true);
        Post postArray = new Post(true);

        output.putAll(getDataScraper());
        output.put("metadata", this.getMetadata());

        try {
            postArray.put(this.scraperName, output);
        } catch (Exception e) {
            DAO.severe("check internet connection");
        }

        return output;
    }

    public Post getDataScraper() {
        Post outputOfScraper = null;

        if("cache".equals(this.source)) {
            outputOfScraper = this.getCache();
        } else {
            outputOfScraper = this.getResults();
        }
        return outputOfScraper;
    }

    public Post getCache() {
        this.setCacheMap();
        DAO.SearchLocalMessages indexScrape = new DAO.SearchLocalMessages (this.cacheMap,
                this.order, this.resultCount);
        this.hits = this.hits + indexScrape.postList.getHits();

        // Add scraper name
        Post postArray = new Post(true);
        postArray.put(this.scraperName, indexScrape.postList.toArray());

        return postArray;
    }

    public Post getResults() {
        try {
            return getDataFromConnection();
        } catch (IOException e) {
            DAO.severe("Error on connection to url:" + this.prepareSearchUrl("all"));
            return new Post(true);
        }
    }

    public Post getDataFromConnection(String url, String type) throws IOException {
        // This adds to hits count even if connection fails
        this.hits++;
        ClientConnection connection = new ClientConnection(url, "");
        BufferedReader br;
        Post postArray = null;
        try {

            // get instance of bufferReader
            br = getHtml(connection);
            postArray = this.scrape(br, type, url);

        } catch (Exception e) {
            DAO.trace(e);
            postArray = new Post(true);
        } finally {
            connection.close();
        }
        return postArray;
    }

    public Post getDataFromConnection(String url) throws IOException {
        return getDataFromConnection(url, "all");
    }

    public Post getDataFromConnection() throws IOException {
        String url = this.prepareSearchUrl("all");
        return getDataFromConnection(url, "all");
    }

    public BufferedReader getHtml(ClientConnection connection) {
        if (connection.getInputStream() == null) {
            return null;
        }
        BufferedReader br = new BufferedReader(
                new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
        return br;
    }

    protected abstract Post scrape(BufferedReader br, String type, String url);

    protected Post putData(Post typeArray, String key, PostTimeline postList) {
        if(!"cache".equals(this.source)) {
            //TODO: base it on SourceType
            postList.writeToIndex();
        }
        return this.putData(typeArray, key, postList.toArray());
    }

    protected Post putData(Post typeArray, String key, JSONArray postList) {
        this.count = this.count + postList.length();
        typeArray.put(key, postList);
        return typeArray;
    }

    protected Post getMetadata() {
        Post metadata = new Post(true);

        metadata.put("hits", this.hits);
        metadata.put("count", this.count);
        metadata.put("scraper", this.scraperName);
        metadata.put("input_parameters", this.extra);

        //TODO: implement these
        //metadata.put("provider_type", this.providerType);
        //metadata.put("source_type", this.sourceType);

        return metadata;
    }

    public String bufferedReaderToString(BufferedReader br) throws IOException {
    StringBuilder everything = new StringBuilder();
    String line;
    while( (line = br.readLine()) != null) {
       everything.append(line);
    }
    return everything.toString();
    }
} public class ClientConnection {

    //public static String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36";
    public static String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"; // see https://developers.whatismybrowser.com/useragents/parse/607126-firefox-windows-gecko
    
    public  static final String CHARSET = "UTF-8";
    private static final byte LF = 10;
    private static final byte CR = 13;
    public static final byte[] CRLF = {CR, LF};
    private static final boolean debugLog = DAO.getConfig("flag.debug.redirect_unshortener", "false").equals("true");

    private static RequestConfig defaultRequestConfig = RequestConfig.custom()
            .setSocketTimeout(60000)
            .setConnectTimeout(60000)
            .setConnectionRequestTimeout(60000)
            .setContentCompressionEnabled(true)
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
            .build();

    private final static CloseableHttpClient httpClient = getClosableHttpClient();

    private BufferedInputStream inputStream;
    private CloseableHttpResponse httpResponse;

    private static class TrustAllHostNameVerifier implements HostnameVerifier {
		public boolean verify(String hostname, SSLSession session) {
			return true;
		}
	}

	/**
     * GET request
     * @param urlstring
     * @param useAuthentication
     * @throws IOException
     */
    public ClientConnection(String urlstring, String q) throws IOException {
        HttpRequestBase request = new HttpGet(urlstring);
        request.setHeader("User-Agent", USER_AGENT);
        request.setHeader("Cache-Control", "max-age=0");
        request.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
        request.setHeader("Accept-Language", "en-US,en;q=0.8,en-GB;q=0.6,es;q=0.4");
        if (q != null && q.length() > 0) request.setHeader("Referer", "https://twitter.com/" + q);
        request.setHeader("X-Requested-With", "XMLHttpRequest");
        this.executeRequest(request);
    }

    /**
     * POST request
     * @param urlstring
     * @param map
     * @param useAuthentication
     * @throws ClientProtocolException
     * @throws IOException
     */
    public ClientConnection(String urlstring, Map<String, byte[]> map) throws ClientProtocolException, IOException {
    	HttpRequestBase request = new HttpPost(urlstring);
        MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
        entityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
        for (Map.Entry<String, byte[]> entry: map.entrySet()) {
            entityBuilder.addBinaryBody(entry.getKey(), entry.getValue());
        }
        ((HttpPost) request).setEntity(entityBuilder.build());
        request.setHeader("User-Agent", USER_AGENT);
        this.executeRequest(request);
    }

    public final static CloseableHttpClient getClosableHttpClient() {
        boolean trustAllCerts = !"none".equals(DAO.getConfig("httpsclient.trustselfsignedcerts", "peers"))
                && ("all".equals(DAO.getConfig("httpsclient.trustselfsignedcerts", "peers")));
        System.setProperty("http.keepAlive", "true"); // ensure that the keep alive strategy is used when default is built
        System.setProperty("http.maxConnections", "200");
        System.setProperty("http.agent", USER_AGENT);
        return HttpClients.custom()
                .useSystemProperties() // makes it possible to inject properties
                .setConnectionManager(getConnctionManager(trustAllCerts))
                .setDefaultRequestConfig(defaultRequestConfig)
                .setMaxConnPerRoute(200)
                .setMaxConnTotal(500)
                .build();
    }

    public InputStream getInputStream() {
        return this.inputStream;
    }

    /**
     * get a connection manager
     * @param trustAllCerts allow opportunistic encryption if needed
     * @return
     */
    private static HttpClientConnectionManager getConnctionManager(boolean trustAllCerts) {

    	Registry<ConnectionSocketFactory> socketFactoryRegistry = null;
    	if(trustAllCerts){
	    	try {
	    		SSLConnectionSocketFactory trustSelfSignedSocketFactory = new SSLConnectionSocketFactory(
				    		new SSLContextBuilder().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build(),
				            new TrustAllHostNameVerifier());
				socketFactoryRegistry = RegistryBuilder
		                .<ConnectionSocketFactory> create()
		                .register("http", new PlainConnectionSocketFactory())
		                .register("https", trustSelfSignedSocketFactory)
		                .build();
			} catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
				DAO.severe(e);
			}
    	}

    	PoolingHttpClientConnectionManager cm = (trustAllCerts && socketFactoryRegistry != null) ?
        		new PoolingHttpClientConnectionManager(socketFactoryRegistry):
        		new PoolingHttpClientConnectionManager();

        // twitter specific options
        cm.setMaxTotal(200);
        cm.setDefaultMaxPerRoute(20);
        cm.setMaxPerRoute(new HttpRoute(new HttpHost("twitter.com", 80)), 50);
        cm.setMaxPerRoute(new HttpRoute(new HttpHost("twitter.com", 443)), 50);

        return cm;
    }

    private void executeRequest(HttpRequestBase request) throws IOException {

        this.httpResponse = null;
        HttpContext context = HttpClientContext.create();
        try {
            this.httpResponse = httpClient.execute(request, context);
        } catch (UnknownHostException e) {
            request.reset();
            throw new IOException("client connection failed: unknown host " + request.getURI().getHost());
        } catch (SocketTimeoutException e){
        	request.reset();
        	throw new IOException("client connection timeout for request: " + request.getURI());
        } catch (SSLHandshakeException e){
        	request.reset();
        	throw new IOException("client connection handshake error for domain " + request.getURI().getHost() + ": " + e.getMessage());
        } catch (Throwable e) {
            request.reset();
            throw new IOException("server failed " + request.getMethod() + " request: " + request.getURI().toString() + ": " + e.getMessage());
        }
        HttpEntity httpEntity = this.httpResponse.getEntity();
        if (httpEntity != null) {
            StatusLine sl = this.httpResponse.getStatusLine();
            int status = sl.getStatusCode();
            if (status == 200) {
                try {
                    this.inputStream = new BufferedInputStream(httpEntity.getContent());
                } catch (IOException e) {
                    request.reset();
                    throw e;
                }
                Map<String, List<String>> header = new HashMap<String, List<String>>();
                for (Header h: httpResponse.getAllHeaders()) {
                    List<String> vals = header.get(h.getName());
                    if (vals == null) { vals = new ArrayList<String>(); header.put(h.getName(), vals); }
                    vals.add(h.getValue());
                }
            } else {
                request.reset();
                throw new IOException("client connection to " + request.getURI() + " fail: " + status + ": " + sl.getReasonPhrase());
            }
        } else {
            request.reset();
            throw new IOException("client connection to " + request.getURI() + " fail: no connection");
        }
    }

    /**
     * Get body of HttpEntity as String
     * @param httpREntity
     * @return response body
     * @throws IOException when readLine fails
     */
    public static String getHTML(HttpEntity httpEntity) throws IOException {
        StringBuilder sb = new StringBuilder();
        BufferedReader reader = new BufferedReader(new InputStreamReader(httpEntity.getContent()));
        String content = null;
        while ((content = reader.readLine()) != null)
            sb.append(content);
        return sb.toString();
    }

    /**
     * Get redirect URL from Meta tag in response body
     * @param httpEntity
     * @return redirect URL if any else null
     * @throws IOException when getHTML() fails
     */
    public static String getMetaRedirectURL(HttpEntity httpEntity) throws IOException {
        String html = getHTML(httpEntity);
        html = html.replace("\n", "");  // May have a line break between tags
        if (html.length() == 0)
            return null;
        int indexHttpEquiv = html.toLowerCase().indexOf("http-equiv=\"refresh\"");  // Converting to lowercase because of case insensitivity
        if (indexHttpEquiv < 0) {
            return null;
        }
        html = html.substring(indexHttpEquiv);
        int indexContent = html.toLowerCase().indexOf("content=");
        if (indexContent < 0) {
            return null;
        }
        html = html.substring(indexContent);
        int indexURLStart = html.toLowerCase().indexOf(";url=");
        if (indexURLStart < 0) {
            return null;
        }
        html = html.substring(indexURLStart + 5);
        int indexURLEnd = html.toLowerCase().indexOf("\"");
        if (indexURLEnd < 0) {
            return null;
        }
        return html.substring(0, indexURLEnd);
    }

    /**
     * Find value of location header from the given HttpResponse
     * @param httpResponse
     * @return Value of location field if exists, null otherwise
     */
    public static String getLocationHeader(CloseableHttpResponse httpResponse) {
        for (Header header: httpResponse.getAllHeaders()) {
            if (header.getName().equalsIgnoreCase("location")) {
                return header.getValue();
            }
        }
        return null;
    }

    /**
     * get a redirect for an url: this method shall be called if it is expected that a url
     * is redirected to another url. This method then discovers the redirect.
     * @param urlstring
     * @param useAuthentication
     * @param isGet
     * @return the redirect url for the given urlstring
     * @throws IOException if there is some error with the link
     */
    private final static String getRedirect(String urlstring, boolean isGet) throws IOException {
        HttpRequestBase req;
        if (isGet)
            req = new HttpGet(urlstring);
        else
            req = new HttpPost(urlstring);
        req.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build());
        req.setHeader("User-Agent", USER_AGENT);
        CloseableHttpResponse httpResponse = httpClient.execute(req);
        HttpEntity httpEntity = httpResponse.getEntity();
        if (debugLog) {
            DAO.log("[" + (isGet ? "GET": "POST") + "] Status for " + urlstring + ": " + httpResponse.getStatusLine().toString());
        }
        if (httpEntity != null) {
            int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
            if (300 <= httpStatusCode && httpStatusCode <= 308) {
                String location = getLocationHeader(httpResponse);
                EntityUtils.consumeQuietly(httpEntity);
                if (location != null) {  // A redirect was found
                    return location;
                }
                throw new IOException("redirect for  " + urlstring+ ": no location attribute found");
            } else if (isGet && httpStatusCode != 200) {
                /*
                    In most of the cases when requesting to http://fb.me/*, a GET request results in 400 Bad Request
                    But making a POST request does the job.
                */
                if (debugLog) {
                    DAO.log("GET method failed for " + urlstring + " trying POST");
                }
                return getRedirect(urlstring, false);
            } else {
                if (debugLog)
                    DAO.log("Trying to fetch Meta redirect URL for " + urlstring);
                String metaURL = getMetaRedirectURL(httpEntity);
                EntityUtils.consumeQuietly(httpEntity);
                if (metaURL != null) {  // A URL was found in meta tag
                    if (debugLog) {
                        DAO.log("Fetched Meta redirect URL for " + urlstring + " : " + metaURL);
                    }
                    if (!metaURL.startsWith("http")) {
                        URL u = new URL(new URL(urlstring), metaURL);
                        return u.toString();
                    }
                    return metaURL;
                }
                return urlstring;
            }
        } else {
            throw new IOException("client connection to " + urlstring + " fail: no connection");
        }
    }

    /**
     * get a redirect for an url: this method shall be called if it is expected that a url
     * is redirected to another url. This method then discovers the redirect.
     * @param urlstring
     * @return
     * @throws IOException
     */
    public static String getRedirect(String urlstring) throws IOException {
        return getRedirect(urlstring, true);
    }

    public void close() {
        if (this.httpResponse != null) {
            HttpEntity httpEntity = this.httpResponse.getEntity();
            if (httpEntity != null) EntityUtils.consumeQuietly(httpEntity);
            this.httpResponse = null;
        }
        if (this.inputStream != null) {
            try {this.inputStream.close();} catch (IOException e) {}
            this.inputStream = null;
        }
    }

    public void finalize() {
        this.close();
    }

    public static void download(String source_url, File target_file) {
        try {
            ClientConnection connection = new ClientConnection(source_url, "");
            try {
                OutputStream os = new BufferedOutputStream(new FileOutputStream(target_file));
                int count;
                byte[] buffer = new byte[2048];
                try {
                    while ((count = connection.inputStream.read(buffer)) > 0) os.write(buffer, 0, count);
                } catch (IOException e) {
                	DAO.severe(e);
                } finally {
                    os.close();
                }
            } catch (IOException e) {
            	DAO.severe(e);
            } finally {
                connection.close();
            }
        } catch (IOException e) {
        	DAO.severe(e);
        }
    }

    public static byte[] download(String source_url) throws IOException {
        ClientConnection connection = new ClientConnection(source_url, "");
        if (connection.inputStream == null) return null;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        int count;
        byte[] buffer = new byte[4096];
        IOException ee = null;
        try {
            while ((count = connection.inputStream.read(buffer)) > 0) baos.write(buffer, 0, count);
        } catch (IOException e) {
        	ee = e;
        } finally {
            connection.close();
        }
        if (ee != null) throw ee;
        return baos.toByteArray();
    }

    public int getStatusCode() {
        return this.httpResponse.getStatusLine().getStatusCode();
    }

    public static void main(String[] args) {
        // lets make a read-world test to find out which closing-methods cause less traffic
        String url[] = new String[]{
                "http://yacy.net/index.html",
                "http://yacy.net/de/index.html",
                "http://yacy.net/de/Bildschirmfotos.html",
                "http://yacy.net/de/Lehrfilme.html",
                "http://yacy.net/de/Philosophie.html",
                "http://yacy.net/de/Suchportal.html",
                "http://yacy.net/de/Anwendungen.html",
                "http://loklak.org/index.html",
                "http://yacy.net/de/API.html",
                "http://yacy.net/de/Technik.html",
                "http://yacy.net/de/Mitmachen.html",
                "http://yacy.net/en/index.html",
                "http://yacy.net/en/Screenshots.html",
                "http://yacy.net/en/Tutorials.html",
                "http://yacy.net/en/Philosophy.html",
                "http://yacy.net/en/Searchportal.html",
                "http://yacy.net/en/Applications.html",
                "http://yacy.net/en/API.html",
                "http://yacy.net/en/Technology.html",
                "http://yacy.net/en/Join.html"
        };
        long s = System.currentTimeMillis();
        for (int i = 0; i< url.length; i++) {
            try {
                long start = System.currentTimeMillis();
                byte[] b = download(url[i]);
                long stop = System.currentTimeMillis();
                long t = stop - start;
                System.out.println(b.length + " bytes in " + t + " milliseconds, " + (b.length * 1000 / (t + 1)) + " bytes/s");
                //if (i == 7) try {Thread.sleep(30000);} catch (InterruptedException e) {}
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        System.out.println("total time: " + (System.currentTimeMillis() - s) + " milliseconds");
    }
} public class AccountFactory extends AbstractIndexFactory<AccountEntry> implements IndexFactory<AccountEntry> {
    
    public AccountFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) {
        super(elasticsearch_client, index_name, cacheSize, existSize);
    }

    @Override
    public AccountEntry init(JSONObject map) throws IOException {
        return new AccountEntry(map);
    }
    
} public class BulkWriteResult {
    
    private Map<String, String> errors;
    private Set<String> created;
    
    public BulkWriteResult() {
        this.errors = new LinkedHashMap<>();
        this.created = new LinkedHashSet<>();
    }
    
    public Map<String, String> getErrors() {
        return this.errors;
    }
    
    public Set<String> getCreated() {
        return this.created;
    }
    
} public class ElasticsearchClient {

    private static long throttling_time_threshold = 2000L; // if updates to the elasticsearch index exceeds this time, a throttling will be applied
    private static long throttling_ops_threshold = 1000L; // messages per second low limit
    private static double throttling_factor = 0.5d; // factor applied on update duration if both thresholds are passed

    public final static BulkWriteResult EMPTY_BULK_RESULT = new BulkWriteResult();

    private Node elasticsearchNode;
    private Client elasticsearchClient;

    /**
     * create a elasticsearch transport client (remote elasticsearch)
     * @param addresses an array of host:port addresses
     * @param clusterName
     */
    public ElasticsearchClient(final String[] addresses, final String clusterName) {
        // create default settings and add cluster name
        Settings.Builder settings = Settings.builder()
                .put("cluster.name", clusterName)
                .put("cluster.routing.allocation.enable", "all")
                .put("cluster.routing.allocation.allow_rebalance", "true");
        // create a client
        TransportClient tc = TransportClient.builder()
                .settings(settings.build())
                .build();
        for (String address: addresses) {
            String a = address.trim();
            int p = a.indexOf(':');
            if (p >= 0) try {
                InetAddress i = InetAddress.getByName(a.substring(0, p));
                int port = Integer.parseInt(a.substring(p + 1));
                tc.addTransportAddress(new InetSocketTransportAddress(i, port));
            } catch (UnknownHostException e) {
            	DAO.severe(e);
            }
        }
        this.elasticsearchClient = tc;
    }

    /**
     * create a elasticsearch node client (embedded elasticsearch)
     * @param settings
     */
    public ElasticsearchClient(final Settings.Builder settings) {
        // create a node
        this.elasticsearchNode = NodeBuilder.nodeBuilder().local(false).settings(settings).node();
        // create a client
        this.elasticsearchClient = elasticsearchNode.client();
    }

    public boolean wait_ready(long maxtimemillis, ClusterHealthStatus status) {
        // wait for yellow status
        long start = System.currentTimeMillis();
        boolean is_ready;
        do {
            // wait for yellow status
            ClusterHealthResponse health = elasticsearchClient.admin().cluster().prepareHealth().setWaitForStatus(status).execute().actionGet();
            is_ready = !health.isTimedOut();
            if (!is_ready && System.currentTimeMillis() - start > maxtimemillis) return false;
        } while (!is_ready);
        return is_ready;
    }

    public void createIndexIfNotExists(String indexName, final int shards, final int replicas) {
        // create an index if not existent
        if (!this.elasticsearchClient.admin().indices().prepareExists(indexName).execute().actionGet().isExists()) {
            Settings.Builder settings = Settings.builder()
                    .put("number_of_shards", shards)
                    .put("number_of_replicas", replicas);
            this.elasticsearchClient.admin().indices().prepareCreate(indexName)
                .setSettings(settings)
                .setUpdateAllTypes(true)
                .execute().actionGet();
        } else {
            //LOGGER.debug("Index with name {} already exists", indexName);
        }
    }

    public void setMapping(String indexName, File json) {
        try {
            this.elasticsearchClient.admin().indices().preparePutMapping(indexName)
                .setSource(new String(Files.readAllBytes(json.toPath()), StandardCharsets.UTF_8))
                .setUpdateAllTypes(true)
                .setType("_default_")
                .execute()
                .actionGet();
        } catch (Throwable e) {
        	DAO.severe(e);
        };
    }

    public String pendingClusterTasks() {
        PendingClusterTasksResponse r = this.elasticsearchClient.admin().cluster().preparePendingClusterTasks().get();
        return r.prettyPrint();
    }

    public String clusterStats() {
        ClusterStatsResponse r = this.elasticsearchClient.admin().cluster().prepareClusterStats().get();
        return r.toString();
    }

    public Map<String, String> nodeSettings() {
        return this.elasticsearchClient.settings().getAsMap();
    }

    /**
     * Close the connection to the remote elasticsearch client. This should only be called when the application is
     * terminated.
     * Please avoid to open and close the ElasticsearchClient for the same cluster and index more than once.
     * To avoid that this method is called more than once, the elasticsearch_client object is set to null
     * as soon this was called the first time. This is needed because the finalize method calls this
     * method as well.
     */
    public void close() {
        if (this.elasticsearchClient != null) {
            this.elasticsearchClient.close();
            this.elasticsearchClient = null;
        }
        if (this.elasticsearchNode != null) {
            this.elasticsearchNode.close();
            this.elasticsearchNode = null;
        }
    }


    /**
     * A finalize method is added to ensure that close() is always called.
     */
    public void finalize() {
        this.close(); // will not cause harm if this is the second call to close()
    }

    /**
     * Get the number of documents in the search index
     *
     * @return the count of all documents in the index
     */
    public long count(String indexName) {
        return count(QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery()), indexName);
    }


    /**
     * Get the number of documents in the search index for a given search query
     *
     * @param q
     *            the query
     * @return the count of all documents in the index which matches with the query
     */
    private long count(final QueryBuilder q, final String indexName) {
        SearchResponse response =
            elasticsearchClient.prepareSearch(indexName).setQuery(q).setSize(0).execute().actionGet();
        return response.getHits().getTotalHits();
    }

    public long count(final String index, final String histogram_timefield, final long millis) {
        try {
            SearchResponse response = elasticsearchClient.prepareSearch(index)
                .setSize(0)
                .setQuery(millis <= 0 ? QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery()) : QueryBuilders.rangeQuery(histogram_timefield).from(new Date(System.currentTimeMillis() - millis)))
                .execute()
                .actionGet();
            return response.getHits().getTotalHits();
        } catch (Throwable e) {
        	DAO.severe(e);
            return 0;
        }
    }

    public long countLocal(final String index, final String provider_hash) {
        try {
            SearchResponse response = elasticsearchClient.prepareSearch(index)
                .setSize(0)
                .setQuery(QueryBuilders.matchQuery("provider_hash", provider_hash))
                .execute()
                .actionGet();
            return response.getHits().getTotalHits();
        } catch (Throwable e) {
        	DAO.severe(e);
            return 0;
        }
    }

    /**
     * Get the document for a given id. If you don't know the typeName of the index, then it is not recommended
     * to use this method. You can set typeName to null and get the correct answer, but you still need the information
     * in which type the document was found if you want to call this API with the type afterwards. In such a case,
     * use the method getType() which returns null if the document does not exist and the type name if the document exist.
     * DO NOT USE THIS METHOD if you call getType anyway. I.e. replace a code like
     * if (exist(id()) {
     *   String type = getType(id);
     *   ...
     * }
     * with
     * String type = getType(id);
     * if (type != null) {
     *   ...
     * }
     *
     * @param indexName
     *            the name of the index
     * @param typeName
     *            the type name, can be set to NULL for all types (see also: getType())
     * @param id
     *            the unique identifier of a document
     * @return the document, if it exists or null otherwise;
     */
    public boolean exist(String indexName, String typeName, final String id) {
        GetResponse getResponse = elasticsearchClient
                .prepareGet(indexName, typeName, id)
                .setOperationThreaded(false)
                .setFields(new String[]{})
                .execute()
                .actionGet();
        return getResponse.isExists();
    }

    public Set<String> existBulk(String indexName, String typeName, final Collection<String> ids) {
        if (ids == null || ids.size() == 0) return new HashSet<>();
        MultiGetResponse multiGetItemResponses = elasticsearchClient.prepareMultiGet()
                .add(indexName, typeName, ids)
                .get();
        Set<String> er = new HashSet<>();
        for (MultiGetItemResponse itemResponse : multiGetItemResponses) {
            GetResponse response = itemResponse.getResponse();
            if (response.isExists()) {
                er.add(response.getId());
            }
        }
        return er;
    }

    /**
     * Delete a document for a given id.
     * ATTENTION: deleted documents cannot be re-inserted again if version number
     * checking is used and the new document does not comply to the version number
     * rule. The information which document was deleted persists for one minute and
     * then inserting documents with the same version number as before is possible.
     * To modify this behavior, change the configuration setting index.gc_deletes
     *
     * @param id
     *            the unique identifier of a document
     * @return true if the document existed and was deleted, false otherwise
     */
    public boolean delete(String indexName, String typeName, final String id) {
        return elasticsearchClient.prepareDelete(indexName, typeName, id).execute().actionGet().isFound();
    }

    /**
     * Delete a list of documents for a given set of ids
     * ATTENTION: read about the time-out of version number checking in the method above.
     *
     * @param ids
     *            a map from the unique identifier of a document to the document type
     * @return the number of deleted documents
     */
    private int deleteBulk(String indexName, Map<String, String> ids) {
        // bulk-delete the ids
        if (ids == null || ids.size() == 0) return 0;
        BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
        for (Map.Entry<String, String> id : ids.entrySet()) {
            bulkRequest.add(new DeleteRequest().id(id.getKey()).index(indexName).type(id.getValue()));
        }
        bulkRequest.execute().actionGet();
        return ids.size();
    }

    /**
     * Delete documents using a query. Check what would be deleted first with a normal search query!
     * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed
     * in later versions. Instead, there is a plugin which iterates over search results,
     * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html
     * We simulate the same behaviour here without the need of that plugin.
     *
     * @param q
     * @return delete document count
     */
    public int deleteByQuery(String indexName, final QueryBuilder q) {
        Map<String, String> ids = new TreeMap<>();
        // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`"
        @SuppressWarnings("deprecation")
        SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN)
            .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet();
        while (true) {
            // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the
            // scroll
            for (SearchHit hit : response.getHits().getHits()) {
                ids.put(hit.getId(), hit.getType());
            }
            response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000))
                .execute().actionGet();
            // termination
            if (response.getHits().getHits().length == 0)
                break;
        }
        return deleteBulk(indexName, ids);
    }

    /**
     * Read a json document from the search index for a given id.
     * Elasticsearch reads the '_source' field and parses the content as json.
     *
     * @param id
     *            the unique identifier of a document
     * @return the document as json, matched on a Map<String, Object> object instance
     */
    public Map<String, Object> readMap(String indexName, final String id) {
        GetResponse response = elasticsearchClient.prepareGet(indexName, null, id).execute().actionGet();
        Map<String, Object> map = getMap(response);
        return map;
    }

    private static Map<String, Object> getMap(GetResponse response) {
        Map<String, Object> map = null;
        if (response.isExists() && (map = response.getSourceAsMap()) != null) {
            map.put("$type", response.getType());
        }
        return map;
    }

    /**
     * Write a json document into the search index. The id must be calculated by the calling environment.
     * This id should be unique for the json. The best way to calculate this id is, to use an existing
     * field from the jsonMap which contains a unique identifier for the jsonMap.
     *
     * @param indexName
     *            the name of the index
     * @param jsonMap
     *            the json document to be indexed in elasticsearch
     * @param typeName
     *            the type of the index
     * @param id
     *            the unique identifier of a document
     * @return true if the document with given id did not exist before, false if it existed and was overwritten
     */
    public boolean writeMap(String indexName, final Map<String, Object> jsonMap, String typeName, String id) {
        long start = System.currentTimeMillis();
        // get the version number out of the json, if any is given
        ListenableActionFuture<IndexResponse> future = writeMapInternal(indexName, jsonMap, typeName, id);
        IndexResponse r = future.actionGet();
        // TODO: error handling
        boolean created = r.isCreated(); // true means created, false means updated
        long duration = Math.max(1, System.currentTimeMillis() - start);
        long regulator = 0;
        if (duration > throttling_time_threshold) {
            regulator = (long) (throttling_factor * duration);
            try {Thread.sleep(regulator);} catch (InterruptedException e) {}
        }
        DAO.log("elastic write entry to index " + indexName + ": " + (created ? "created":"updated") + ", " + duration + " ms" + (regulator == 0 ? "" : ", throttled with " + regulator + " ms"));
        return created;
    }

    public void writeMapAsync(String indexName, final Map<String, Object> jsonMap, String typeName, String id) {
        writeMapInternal(indexName, jsonMap, typeName, id);
    }

    private ListenableActionFuture<IndexResponse> writeMapInternal(String indexName, final Map<String, Object> jsonMap, String typeName, String id) {
        // get the version number out of the json, if any is given
        Long version = (Long) jsonMap.remove("_version");
        // put this to the index
        ListenableActionFuture<IndexResponse> future = elasticsearchClient.prepareIndex(indexName, typeName, id).setSource(jsonMap)
            .setVersion(version == null ? 1 : version.longValue())
            .setVersionType(version == null ? VersionType.FORCE : VersionType.EXTERNAL)
            .execute();
        if (version != null) jsonMap.put("_version", version); // to prevent side effects
        // documentation about the versioning is available at
        // https://www.elastic.co/blog/elasticsearch-versioning-support
        return future;
    }

    /**
     * bulk message write
     * @param jsonMapList
     *            a list of json documents to be indexed
     * @param indexName
     *            the name of the index
     * @param typeName
     *            the type of the index
     * @return a list with error messages.
     *            The key is the id of the document, the value is an error string.
     *            The method was only successful if this list is empty.
     *            This must be a list, because keys may appear several times.
     */
    public BulkWriteResult writeMapBulk(final String indexName, final List<BulkWriteEntry> jsonMapList) {
        long start = System.currentTimeMillis();
        ListenableActionFuture<BulkResponse> future = writeMapBulkInternal(indexName, jsonMapList);
        BulkResponse bulkResponse = future.actionGet();
        BulkWriteResult result = new BulkWriteResult();
        for (BulkItemResponse r: bulkResponse.getItems()) {
            String id = r.getId();
            ActionWriteResponse response = r.getResponse();
            if (response instanceof IndexResponse) {
                if (((IndexResponse) response).isCreated()) result.getCreated().add(id);
            }
            String err = r.getFailureMessage();
            if (err != null) {
                result.getErrors().put(id, err);
            }
        }
        long duration = Math.max(1, System.currentTimeMillis() - start);
        long regulator = 0;
        long ops = result.getCreated().size() * 1000 / duration; // this is the number of updates per second
        if (duration > throttling_time_threshold && ops < throttling_ops_threshold) {
            regulator = (long) (throttling_factor * duration);
            try {Thread.sleep(regulator);} catch (InterruptedException e) {}
        }
        DAO.log("elastic write bulk to index " + indexName + ": " + jsonMapList.size() + " entries, " + result.getCreated().size() + " created, " + result.getErrors().size() + " errors, " + duration + " ms" + (regulator == 0 ? "" : ", throttled with " + regulator + " ms") + ", " + ops + " objects/second");
        if(result.getErrors().size() > 0) {
            DAO.severe("Errors faced while indexing:");
            for(String key : result.getErrors().keySet()) {
                DAO.severe(result.getErrors().get(key));
            }
        }
        return result;
    }

    public void writeMapBulkAsync(final String indexName, final List<BulkWriteEntry> jsonMapList) {
        writeMapBulkInternal(indexName, jsonMapList);
    }

    private ListenableActionFuture<BulkResponse> writeMapBulkInternal(final String indexName, final List<BulkWriteEntry> jsonMapList) {
        final BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
        for (BulkWriteEntry be: jsonMapList) {
            if (be.getId() == null) continue;
            bulkRequest.add(
                    elasticsearchClient.prepareIndex(indexName, be.getType(), be.getId()).setSource(be.getJsonMap())
                        .setVersion(be.getVersion() == null ? 1 : be.getVersion().longValue())
                        .setVersionType(be.getVersion() == null ? VersionType.FORCE : VersionType.EXTERNAL));
        }

        final ListenableActionFuture<BulkResponse> future = bulkRequest.execute();
        return future;
    }

    public Map<String, Object> query(final String indexName, final String fieldKey, final String fieldValue) {
        if (fieldKey == null || fieldValue.length() == 0) return null;
        // prepare request
        BoolQueryBuilder query = QueryBuilders.boolQuery();
        query.filter(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(fieldKey, fieldValue)));

        SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName)
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setQuery(query)
                .setFrom(0)
                .setSize(1)
                .setTerminateAfter(1);

        // get response
        SearchResponse response = request.execute().actionGet();

        // evaluate search result
        SearchHit[] hits = response.getHits().getHits();
        if (hits.length == 0) return null;
        assert hits.length == 1;

        Map<String, Object> map = hits[0].getSource();

        return map;
    }

    public Query query(final String indexName, final QueryBuilder queryBuilder, String order_field, int timezoneOffset, int resultCount, long histogram_interval, String histogram_timefield, int aggregationLimit, String... aggregationFields) {
        return new Query(indexName,  queryBuilder, order_field, timezoneOffset, resultCount, histogram_interval, histogram_timefield, aggregationLimit, aggregationFields);
    }

    public Query query(
            final String indexName,
            final QueryBuilder queryBuilder,
            String order,
            int resultCount
    ) {
        if (queryBuilder == null) return null;
        return new Query(indexName, queryBuilder, order, resultCount);
    }

    public class Query {
        private List<Map<String, Object>> result; // a list of search results; each map entry contains all attributes of the search result
        private int hitCount; // number of hits in the search process, not the number of results; the hitCount must be always larger or equal to the result list size
        private Map<String, List<Map.Entry<String, AtomicLong>>> aggregations; // key is the field name and list is a list of key-value pairs, where each key names an entry that the key may contain and the value is a counter of the number of occurrences of that value

        /**
         * Search the local message cache using a elasticsearch query.
         * @param q - the query, for aggregation this which should include a time frame in the form since:yyyy-MM-dd until:yyyy-MM-dd
         * @param order_field - the field to order the results, i.e. Timeline.Order.CREATED_AT
         * @param timezoneOffset - an offset in minutes that is applied on dates given in the query of the form since:date until:date
         * @param resultCount - the number of messages in the result; can be zero if only aggregations are wanted
         * @param dateHistogrammInterval - the date aggregation interval or null, if no aggregation wanted
         * @param aggregationLimit - the maximum count of facet entities, not search results
         * @param aggregationFields - names of the aggregation fields. If no aggregation is wanted, pass no (zero) field(s)
         */
        private Query(final String indexName, final QueryBuilder queryBuilder, String order_field, int timezoneOffset, int resultCount, long histogram_interval, String histogram_timefield, int aggregationLimit, String... aggregationFields) {
            // prepare request
            SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName)
                    .setSearchType(SearchType.QUERY_THEN_FETCH)
                    .setQuery(queryBuilder)
                    .setFrom(0)
                    .setSize(resultCount);
            request.clearRescorers();
            if (resultCount > 0) {
                request.addSort(
                        SortBuilders.fieldSort(order_field)
                            .unmappedType(order_field)
                            .order(SortOrder.DESC)
                        );
            }
            boolean addTimeHistogram = false;
            DateHistogramInterval dateHistogrammInterval = histogram_interval > DateParser.WEEK_MILLIS ? DateHistogramInterval.DAY : histogram_interval > DateParser.HOUR_MILLIS * 3 ? DateHistogramInterval.HOUR : DateHistogramInterval.MINUTE;
            for (String field: aggregationFields) {
                if (field.equals(histogram_timefield)) {
                    addTimeHistogram = true;
                    request.addAggregation(AggregationBuilders.dateHistogram(histogram_timefield).field(histogram_timefield).timeZone("UTC").minDocCount(0).interval(dateHistogrammInterval));
                } else {
                    request.addAggregation(AggregationBuilders.terms(field).field(field).minDocCount(1).size(aggregationLimit));
                }
            }
            // get response
            SearchResponse response = request.execute().actionGet();
            hitCount = (int) response.getHits().getTotalHits();

            // evaluate search result
            //long totalHitCount = response.getHits().getTotalHits();
            SearchHit[] hits = response.getHits().getHits();
            this.result = new ArrayList<Map<String, Object>>(hitCount);
            for (SearchHit hit: hits) {
                Map<String, Object> map = hit.getSource();
                this.result.add(map);
            }

            // evaluate aggregation
            // collect results: fields
            this.aggregations = new HashMap<>();
            for (String field: aggregationFields) {
                if (field.equals(histogram_timefield)) continue; // this has special handling below
                Terms fieldCounts = response.getAggregations().get(field);
                List<Bucket> buckets = fieldCounts.getBuckets();
                // aggregate double-tokens (matching lowercase)
                Map<String, Long> checkMap = new HashMap<>();
                for (Bucket bucket: buckets) {
                    String key = bucket.getKeyAsString().trim();
                    if (key.length() > 0) {
                        String k = key.toLowerCase();
                        Long v = checkMap.get(k);
                        checkMap.put(k, v == null ? bucket.getDocCount() : v + bucket.getDocCount());
                    }
                }
                ArrayList<Map.Entry<String, AtomicLong>> list = new ArrayList<>(buckets.size());
                for (Bucket bucket: buckets) {
                    String key = bucket.getKeyAsString().trim();
                    if (key.length() > 0) {
                        Long v = checkMap.remove(key.toLowerCase());
                        if (v == null) continue;
                        list.add(new AbstractMap.SimpleEntry<String, AtomicLong>(key, new AtomicLong(v)));
                    }
                }
                aggregations.put(field, list);
                //if (field.equals("place_country")) {
                    // special handling of country aggregation: add the country center as well
                //}
            }
            // date histogram:
            if (addTimeHistogram) {
                InternalHistogram<InternalHistogram.Bucket> dateCounts = response.getAggregations().get(histogram_timefield);
                ArrayList<Map.Entry<String, AtomicLong>> list = new ArrayList<>();
                for (InternalHistogram.Bucket bucket : dateCounts.getBuckets()) {
                    Calendar cal = Calendar.getInstance(DateParser.UTCtimeZone);
                    org.joda.time.DateTime k = (org.joda.time.DateTime) bucket.getKey();
                    cal.setTime(k.toDate());
                    cal.add(Calendar.MINUTE, -timezoneOffset);
                    long docCount = bucket.getDocCount();
                    Map.Entry<String, AtomicLong> entry = new AbstractMap.SimpleEntry<>(
                        (dateHistogrammInterval == DateHistogramInterval.DAY ? DateParser.dayDateFormat : DateParser.minuteDateFormat).format(cal.getTime()),
                        new AtomicLong(docCount));
                    list.add(entry);
                }
                aggregations.put(histogram_timefield, list);
            }
        }

        private Query(final String indexName, QueryBuilder queryBuilder, String order_field, int resultCount) {
            //TODO: sort data using order_field
            // prepare request
            SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName)
                    .setSearchType(SearchType.QUERY_THEN_FETCH)
                    .setQuery(queryBuilder)
                    .setFrom(0)
                    .setSize(resultCount);
            request.clearRescorers();

            // get response
            SearchResponse response = request.execute().actionGet();
            hitCount = (int) response.getHits().getTotalHits();
            // evaluate search result
            SearchHit[] hits = response.getHits().getHits();
            this.result = new ArrayList<Map<String, Object>>(hitCount);
            for (SearchHit hit: hits) {
                Map<String, Object> map = hit.getSource();
                this.result.add(map);
            }

        }

        /**
         * combine this query with another query
         * @param other the other search result
         */
        public void add(Query other) {

            // add new hits
            otherLoop: for (Map<String, Object> otherEntry: other.result) {
                String otherID = (String) otherEntry.get("id_str");
                if (otherID == null) continue otherLoop;
                ownLoop: for (Map<String, Object> ownEntry: this.result) {
                    String ownID = (String) ownEntry.get("id_str");
                    if (ownID == null) continue ownLoop;
                    if (ownID.equals(otherID)) continue otherLoop;
                }
                this.result.add(otherEntry); // this is a new one
            }

            // check overall hits
            this.hitCount = Math.max(Math.max(this.hitCount, other.hitCount), this.result.size());

            // expand the aggregations
            for (Map.Entry<String, List<Map.Entry<String, AtomicLong>>> otherAggregationEntry: other.aggregations.entrySet()) {
                String key = otherAggregationEntry.getKey();
                List<Map.Entry<String, AtomicLong>> otherList = otherAggregationEntry.getValue();
                if (!this.aggregations.containsKey(key)) {
                    this.aggregations.put(key, otherList);
                    continue;
                }

                List<Map.Entry<String, AtomicLong>> ownList = this.aggregations.get(key);
                otherLoop: for (Map.Entry<String, AtomicLong> otherPair: otherList) {
                    for (Map.Entry<String, AtomicLong> ownPair: ownList) {
                        if (ownPair.getKey().equals(otherPair.getKey())) {
                            ownPair.getValue().set(Math.max(ownPair.getValue().get(), otherPair.getValue().get())); // probably this is not correct but the correct value is also not computable. This is the lower approximation of the correct value.
                            continue otherLoop;
                        }
                        ownList.add(otherPair); // because we did not see the key we can just add this here
                    }
                }
            }
        }

        public List<Map<String, Object>> getResult() {
            return this.result;
        }

        public int getHitCount() {
            return this.hitCount;
        }

        public Map<String, List<Map.Entry<String, AtomicLong>>> getAggregations() {
            return this.aggregations;
        }

    }


    /**
     * Search the local message cache using a elasticsearch query.
     * @param q - the query, can be empty for a matchall-query
     * @param resultCount - the number of messages in the result
     * @param sort_field - the field name to sort the result list, i.e. "query_first"
     * @param sort_order - the sort order (you want to use SortOrder.DESC here)
     */
    public ResultList<Map<String, Object>> fuzzyquery(final String indexName, final String fieldName, final String q, final int resultCount, final String sort_field, final String default_sort_type, final SortOrder sort_order, final Date since, final Date until, final String range_field) {

        // prepare request
        BoolQueryBuilder suggest = QueryBuilders.boolQuery();
        if (q != null && q.length() > 0) {
            suggest.should(QueryBuilders.fuzzyQuery(fieldName, q).fuzziness(Fuzziness.fromEdits(2)));
            suggest.should(QueryBuilders.moreLikeThisQuery(fieldName).like(q));
            suggest.should(QueryBuilders.matchPhrasePrefixQuery(fieldName, q));
            if (q.indexOf('*') >= 0 || q.indexOf('?') >= 0) suggest.should(QueryBuilders.wildcardQuery(fieldName, q));
            suggest.minimumNumberShouldMatch(1);
        }

        BoolQueryBuilder query;

        if (range_field != null && range_field.length() > 0 && (since != null || until != null)) {
            query = QueryBuilders.boolQuery();
            if (q.length() > 0) query.filter(suggest);
            RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(range_field);
            if (since != null) rangeQuery.from(since).includeLower(true);
            if (until != null) rangeQuery.to(until).includeUpper(true);
            query.filter(rangeQuery);
        } else {
            query = suggest;
        }

        SearchRequestBuilder request = this.elasticsearchClient.prepareSearch(indexName)
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setQuery(query)
                .setFrom(0)
                .setSize(resultCount)
                .addSort(
                        SortBuilders.fieldSort(sort_field)
                        .unmappedType(default_sort_type)
                        .order(sort_order));

        // get response
        SearchResponse response = request.execute().actionGet();

        // evaluate search result
        //long totalHitCount = response.getHits().getTotalHits();
        SearchHits rhits = response.getHits();
        //long totalHits = rhits.getTotalHits();
        ResultList<Map<String, Object>> result = new ResultList<Map<String, Object>>();
        SearchHit[] hits = rhits.getHits();
        for (SearchHit hit: hits) {
            Map<String, Object> map = hit.getSource();
            result.add(map);
        }
        result.setHits(rhits.getTotalHits());

        return result;
    }

    public List<Map<String, Object>> queryWithConstraints(final String indexName, final String fieldName, final String fieldValue, final Map<String, String> constraints, boolean latest) throws IOException {
        SearchRequestBuilder request = this.elasticsearchClient.prepareSearch(indexName)
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setFrom(0);

        BoolQueryBuilder bFilter = QueryBuilders.boolQuery();
        bFilter.filter(QueryBuilders.constantScoreQuery(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(fieldName, fieldValue))));
        for (Object o : constraints.entrySet()) {
            @SuppressWarnings("rawtypes")
            Map.Entry entry = (Map.Entry) o;
            bFilter.filter(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery((String) entry.getKey(), ((String) entry.getValue()).toLowerCase())));
        }
        request.setQuery(bFilter);

        // get response
        SearchResponse response = request.execute().actionGet();

        // evaluate search result
        ArrayList<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits) {
            Map<String, Object> map = hit.getSource();
            result.add(map);
        }

        return result;
    }

    public LinkedHashMap<String, Long> fullDateHistogram(final String indexName, int timezoneOffset, String histogram_timefield) {
        // prepare request
        SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName)
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery()))
                .setFrom(0)
                .setSize(0);
        request.clearRescorers();
        request.addAggregation(AggregationBuilders.dateHistogram(histogram_timefield).field(histogram_timefield).timeZone("UTC").minDocCount(1).interval(DateHistogramInterval.DAY));

        // get response
        SearchResponse response = request.execute().actionGet();

        // evaluate date histogram:
        InternalHistogram<InternalHistogram.Bucket> dateCounts = response.getAggregations().get(histogram_timefield);
        LinkedHashMap<String, Long> list = new LinkedHashMap<>();
        for (InternalHistogram.Bucket bucket : dateCounts.getBuckets()) {
            Calendar cal = Calendar.getInstance(DateParser.UTCtimeZone);
            org.joda.time.DateTime k = (org.joda.time.DateTime) bucket.getKey();
            cal.setTime(k.toDate());
            cal.add(Calendar.MINUTE, -timezoneOffset);
            long docCount = bucket.getDocCount();
            list.put(DateParser.dayDateFormat.format(cal.getTime()), docCount);
        }
        return list;
    }

    /**
     * Get all time aggregations without any geographical constraints.
     * @param index Name of ES index
     * @param classifierName Classifier for aggregation
     * @param classes Classes to consider
     * @return HashMap with required aggregations
     */
    private final HashMap<String, HashMap<String, Double>> classifierScore(String index, String classifierName, List<String> classes) {
        SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilder(classifierName));
        Terms aggrs = response.getAggregations().get("by_class");
        return getAggregationByClass(aggrs, classes);
    }

    /**
     * Get limited time aggregations without any geographical constraints.
     * @param index Name of ES index
     * @param classifierName Classifier for aggregation
     * @param classes Classes to consider
     * @param startDate Start date for creation of row
     * @param endDate End date for creation of row
     * @return HashMap with required aggregations
     */
    private final HashMap<String, HashMap<String, Double>> classifierScore(
        String index, String classifierName, List<String> classes, String startDate, String endDate) {
        if (startDate == null && endDate == null) {
            return classifierScore(index, classifierName, classes);
        }
        SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilder(classifierName, startDate, endDate));
        Range aggrs = response.getAggregations().get("by_date");
        return getAggregationByTime(aggrs, classes);
    }

    /**
     * Get limited time aggregation by country code for all available countries.
     * @param index Name of ES index
     * @param classifierName Name of classifier
     * @param classes Classes to consider
     * @param startDate Start date for creation of row
     * @param endDate End date for creation of row
     * @return HashMap with required aggregations
     */
    public final HashMap<String, HashMap<String, HashMap<String,Double>>> classifierScoreForCountry(
        String index, String classifierName, List<String> classes, String startDate, String endDate) {
        // No time restrictions
        if (startDate == null && endDate == null) {
            return classifierScoreForCountry(index, classifierName, classes);
        }
        SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilderByCountry(
            "place_country_code", classifierName, startDate, endDate));
        Range aggr = response.getAggregations().get("by_date");
        HashMap<String, HashMap<String, HashMap<String,Double>>> retMap = getAggregationByTimeWithCountries(aggr, classes);
        // Put global aggregation
        retMap.put("GLOBAL", classifierScore(index, classifierName, classes, startDate, endDate));
        return retMap;
    }

    /**
     * Get all time aggregation by country for all available countries.
     * @param index Name of ES index
     * @param classifierName Name of classifier
     * @param classes Classes to consider
     * @return HashMap with required aggregations
     */
    private final HashMap<String, HashMap<String, HashMap<String,Double>>> classifierScoreForCountry(String index, String classifierName, List<String> classes) {
        SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilderByCountry("place_country_code", classifierName));
        Terms aggr = response.getAggregations().get("by_country");
        HashMap<String, HashMap<String, HashMap<String,Double>>> retMap = getAggregationByCountry(aggr, classes);
        // Put global score
        retMap.put("GLOBAL", classifierScore(index, classifierName, classes));
        return retMap;
    }

    /**
     * Get a search response for predefined aggregation task on a specific index.
     * @param index Name of ES index
     * @param aggr Pre-configured AggregationBuilder object
     * @return HashMap with parsed aggregations
     */
    private SearchResponse getAggregationResponse(String index, @SuppressWarnings("rawtypes") AggregationBuilder aggr) {
        return this.elasticsearchClient.prepareSearch(index)
            .setSearchType(SearchType.QUERY_THEN_FETCH)
            .setQuery(QueryBuilders.matchAllQuery())
            .setFrom(0)
            .setSize(0)
            .addAggregation(aggr)
            .execute().actionGet();
    }

    /**
     * Get aggregation results for each class as sum, avg and doc count.
     * @param aggr Terms aggregation object
     * @param classes Classes to be taken into account
     * @return HashMap with required aggregations
     */
    private static HashMap<String, HashMap<String, Double>> getAggregationByClass(Terms aggr, List<String> classes) {
        HashMap<String, HashMap<String, Double>> retMap = new HashMap<>();
        for (Terms.Bucket bucket : aggr.getBuckets()) {
            String key = bucket.getKeyAsString();
            if (!classes.contains(key)) {
                continue;
            }
            long docCount = bucket.getDocCount();
            Sum sum = bucket.getAggregations().get("sum_probability");
            Avg avg = bucket.getAggregations().get("avg_probability");
            HashMap<String, Double> map = new HashMap<>();
            map.put("count", (double) docCount);
            map.put("sum", sum.getValue());
            map.put("avg", avg.getValue());
            retMap.put(key,  map);
        }
        return retMap;
    }

    /**
     * Get aggregation result for all countries.
     * @param aggr Terms aggregation object
     * @param classes Classes to consider
     * @return HashMap with parsed aggregations
     */
    private static HashMap<String, HashMap<String, HashMap<String, Double>>> getAggregationByCountry(Terms aggr, List<String> classes) {
        HashMap<String, HashMap<String, HashMap<String, Double>>> retMap = new HashMap<>();
        for (Terms.Bucket bucket : aggr.getBuckets()) {
            retMap.put(bucket.getKeyAsString(), getAggregationByClass(bucket.getAggregations().get("by_class"), classes));
        }
        return retMap;
    }

    /**
     * Get result for time limited aggregation for all available classes.
     * @param aggr Range aggregation object
     * @param classes Classes to consider
     * @return HashMap with parsed aggregations
     */
    private static HashMap<String, HashMap<String, Double>> getAggregationByTime(Range aggr, List<String> classes) {
        HashMap<String, HashMap<String, Double>> retMap = new HashMap<>();
        for (Range.Bucket bucket : aggr.getBuckets()) {
            retMap = getAggregationByClass(bucket.getAggregations().get("by_class"), classes);
        }
        return retMap;
    }

    /**
     * Get result for time limited aggregation for all countries and required classes.
     * @param aggr Range aggregation object
     * @param classes Classes to consider
     * @return HashMap with parsed results
     */
    private static HashMap<String, HashMap<String, HashMap<String, Double>>> getAggregationByTimeWithCountries(Range aggr, List<String> classes) {
        HashMap<String, HashMap<String, HashMap<String, Double>>> retMap = new HashMap<>();
        for (Range.Bucket bucket : aggr.getBuckets()) {
            retMap = getAggregationByCountry(bucket.getAggregations().get("by_country"), classes);
        }
        return retMap;
    }

    /**
     * Get AggregationBuilder to calculate sum, avg and doc count for given classifier.
     * @param classifierName Name of classifier
     * @return AggregationBuilder with required configuration
     */
    private static TermsBuilder getClassifierAggregationBuilder(String classifierName) {
        String probabilityField = classifierName + "_probability";
        return AggregationBuilders.terms("by_class").field(classifierName)
            .subAggregation(
                AggregationBuilders.avg("avg_probability").field(probabilityField)
            )
            .subAggregation(
                AggregationBuilders.sum("sum_probability").field(probabilityField)
            );
    }

    /**
     * Get AggregationBuilder for specific time range.
     * @param classifierName Name of classifier
     * @param fromDate Start date for creation of row
     * @param toDate End date for creation of row
     * @return AggregationBuilder with required configuration
     */
    private static DateRangeBuilder getClassifierAggregationBuilder(String classifierName, String fromDate, String toDate) {
        return getDateRangeAggregationBuilder("created_at", fromDate, toDate)
            .subAggregation(getClassifierAggregationBuilder(classifierName));
    }

    /**
     * Get AggregationBuilder by country without time constraints.
     * @param fieldName Name of field containing country code
     * @param classifierName Name of field containing country code
     * @return AggregationBuilder with required configuration
     */
    private static TermsBuilder getClassifierAggregationBuilderByCountry(String fieldName, String classifierName) {
        return AggregationBuilders.terms("by_country").field(fieldName)
            .subAggregation(getClassifierAggregationBuilder(classifierName));
    }

    /**
     * Get AggregationBuilder by country for limited time range.
     * @param fieldName Name of field containing country code.
     * @param classifierName Name of classifier
     * @param startDate Start date for creation of row
     * @param endDate End date for creation of row
     * @return AggregationBuilder with required configuration
     */
    private static DateRangeBuilder getClassifierAggregationBuilderByCountry(String fieldName, String classifierName, String startDate, String endDate) {
        return getDateRangeAggregationBuilder("created_at", startDate, endDate)
            .subAggregation(getClassifierAggregationBuilderByCountry(fieldName, classifierName));
    }

    /**
     * Get AggregationBuilder using dateRange with proper limits from parameters.
     * @param fieldName Name of field to time restrict
     * @param fromDate Start date for restriction
     * @param toDate End date for restriction
     * @return AggregationBuilder with required configuration
     */
    private static DateRangeBuilder getDateRangeAggregationBuilder(String fieldName, String fromDate, String toDate) {
        DateRangeBuilder dateRangeAggregation = AggregationBuilders.dateRange("by_date").field(fieldName);
        if (fromDate == null && toDate != null) {
            dateRangeAggregation = dateRangeAggregation.addUnboundedTo(toDate);
        } else if (toDate == null && fromDate != null) {
            dateRangeAggregation = dateRangeAggregation.addUnboundedFrom(fromDate);
        } else {
            dateRangeAggregation = dateRangeAggregation.addRange(fromDate, toDate);
        }
        return dateRangeAggregation;
    }

} public class ImportProfileFactory extends AbstractIndexFactory<ImportProfileEntry> implements IndexFactory<ImportProfileEntry> {

    public ImportProfileFactory(ElasticsearchClient elasticsearch_client, String index_name, int cacheSize, final int existSize) {
        super(elasticsearch_client, index_name, cacheSize, existSize);
    }

    @Override
    public ImportProfileEntry init(JSONObject map) throws IOException {
        return new ImportProfileEntry(map);
    }
} public class MessageFactory extends AbstractIndexFactory<Post> implements IndexFactory<Post> {

    public MessageFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) {
        super(elasticsearch_client, index_name, cacheSize, existSize);
    }

    @Override
    public TwitterTweet init(JSONObject json) {
        return new TwitterTweet(json);
    }
    
} public class QueryFactory extends AbstractIndexFactory<QueryEntry> implements IndexFactory<QueryEntry> {

    public QueryFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) {
        super(elasticsearch_client, index_name, cacheSize, existSize);
    }

    @Override
    public QueryEntry init(JSONObject json) {
        return new QueryEntry(json);
    }
    
} public class UserFactory extends AbstractIndexFactory<UserEntry> implements IndexFactory<UserEntry> {
    
    public UserFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) {
        super(elasticsearch_client, index_name, cacheSize, existSize);
    }

    @Override
    public UserEntry init(JSONObject json) {
        return new UserEntry(json);
    }
    
} public abstract class AbstractObjectEntry extends Post implements ObjectEntry {

    public  final static String TIMESTAMP_FIELDNAME = "timestamp";   // the harvesting time, NOT used for identification
    public  final static String CREATED_AT_FIELDNAME = "created_at"; // the tweet time as embedded in the tweet, not used for identification either
    
    public AbstractObjectEntry() {
    }
    
    public String toString() {
        return super.toString();
    }

    public byte[] toJSONBytes() {
        String s = this.toString();
        return s == null ? null : s.getBytes(StandardCharsets.UTF_8);
    }
    
    // helper methods to write json
    
    public final static DateTimeFormatter utcFormatter = ISODateTimeFormat.dateTime().withZoneUTC();
    
    public static void writeDate(JsonGenerator json, String fieldName, long time) throws IOException {
        json.writeObjectField(fieldName, utcFormatter.print(time));
    }
    
    public static void writeArray(JsonGenerator json, String fieldName, Collection<String> array) throws IOException {
        json.writeArrayFieldStart(fieldName);
        for (String o: array) json.writeObject(o);
        json.writeEndArray();
    }
    
    public static void writeArray(JsonGenerator json, String fieldName, String[] array) throws IOException {
        json.writeArrayFieldStart(fieldName);
        for (String o: array) json.writeObject(o);
        json.writeEndArray();
    }
    
    public static void writeArray(JsonGenerator json, String fieldName, double[] array) throws IOException {
        json.writeArrayFieldStart(fieldName);
        for (double o: array) json.writeObject(o);
        json.writeEndArray();
    }
    
    // helper methods to read json

    public static Date parseDate(Object d) {
        if (d == null) return new Date();
        if (d instanceof Date) return (Date) d;
        if (d instanceof Long) return new Date(((Long) d).longValue());
        if (d instanceof String) return ISODateTimeFormat.dateOptionalTimeParser().parseDateTime((String) d).toDate();
        assert false;
        return new Date();
    }
    
    public static Date parseDate(Object d, Date dflt) {
        if (d == null) return dflt;
        if (d instanceof Long) return new Date(((Long) d).longValue());
        if (d instanceof String) return ISODateTimeFormat.dateOptionalTimeParser().parseDateTime((String) d).toDate();
        if (d instanceof Date) return (Date) d;
        assert false;
        return dflt;
    }

    public static String parseString(Object s) {
        assert s == null || s instanceof String: "type of object s: " + s.getClass().toString();
        return s == null ? "" : (String) s;
    }
    
    public static long parseLong(Object n) {
        if (n instanceof String) {
            return Long.parseLong((String) n);
        }
        assert n == null || n instanceof Number;
        return n == null ? 0 : ((Number) n).longValue();
    }
    
    @SuppressWarnings("unchecked")
    public static LinkedHashSet<String> parseArrayList(Object l) {
        assert l == null || l instanceof String  || l instanceof String[] || l instanceof Collection<?> || l instanceof JSONArray : "unexpected class in parseArrayList:" + l.getClass().getName();
        if (l == null) return new LinkedHashSet<String>();
        if (l instanceof String) {
            LinkedHashSet<String> a = new LinkedHashSet<>();
            a.add((String) l);
            return a;
        }
        if (l instanceof String[]) {
            LinkedHashSet<String> a = new LinkedHashSet<>();
            for (String s: ((String[]) l)) if (s != null) a.add(s);
            return a;
        }
        if (l instanceof LinkedHashSet<?>) {
            LinkedHashSet<String> a = new LinkedHashSet<>();
            for (String s: (LinkedHashSet<String>) l) if (s != null) a.add(s);
            return a;
        }
        if (l instanceof JSONArray) {
            LinkedHashSet<String> a = new LinkedHashSet<String>();
            JSONArray la = (JSONArray) l;
            for (int i = 0; i < la.length(); i++) {
                try {
                    String s = la.getString(i);
                    if (s != null) a.add(s);
                } catch (JSONException e) {}
            }
            return a;
        }
        LinkedHashSet<String> a = new LinkedHashSet<>();
        for (Object s: ((Collection<?>) l)) if (s != null) a.add((String) s);
        return a;
    }

} public class QueryEntry extends AbstractObjectEntry implements ObjectEntry {
    
    public  final static long DAY_MILLIS = 1000L * 60L * 60L * 24L;
    private final static int RETRIEVAL_CONSTANT = 20; // the number of messages that we get with each retrieval at maximum
    
    protected String query;           // the query in the exact way as the user typed it in
    protected int query_length;       // the length in the query, number of characters
    public SourceType source_type;    // the (external) retrieval system where that query was submitted
    protected int timezoneOffset;     // the timezone offset of the user
    protected Date query_first;       // the date when this query was submitted by the user the first time
    protected Date query_last;        // the date when this query was submitted by the user the last time
    protected Date retrieval_last;    // the last time when this query was submitted to the external system
    protected Date retrieval_next;    // the estimated next time when the query should be submitted to get all messages
    protected Date expected_next;     // the estimated next time when one single message will appear
    protected int query_count;        // the number of queries by the user of that query done so far
    protected int retrieval_count;    // the number of retrievals of that query done so far to the external system
    protected long message_period;    // the estimated period length between two messages
    protected int messages_per_day;   // a message frequency based on the last query
    protected long score_retrieval;   // score for the retrieval order
    protected long score_suggest;     // score for the suggest order

    /**
     * This initializer can only be used for first-time creation of a query track.
     * @param query
     * @param timezoneOffset
     * @param message_period
     * @param source_type
     * @throws MalformedURLException
     */
    public QueryEntry(final String query, final int timezoneOffset, final long message_period, final SourceType source_type, final boolean byUserQuery) {
        this.query = query;
        this.query_length = query.length();
        this.timezoneOffset = timezoneOffset;
        this.source_type = source_type;
        this.retrieval_count = 0; // will be set to 1 with first update
        this.message_period = 0; // means: unknown
        this.messages_per_day = 0; // means: unknown
        this.score_retrieval = 0;
        this.score_suggest = 0;
        update(message_period, byUserQuery);
        this.query_first = retrieval_last;
    }

    public QueryEntry(JSONObject json) throws IllegalArgumentException, JSONException {
        init(json);
    }

    public void init(JSONObject json) throws IllegalArgumentException, JSONException {
        this.query = (String) json.get("query");
        this.query_length = (int) parseLong((Number) json.get("query_length"));
        String source_type_string = (String) json.get("source_type");
        if (source_type_string == null) source_type_string = SourceType.TWITTER.toString();
        this.source_type = SourceType.byName(source_type_string);
        this.timezoneOffset = (int) parseLong((Number) json.get("timezoneOffset"));
        Date now = new Date();
        this.query_first = json.has("query_first") ? parseDate(json.get("query_first"), now) : new Date();
        this.query_last = json.has("query_last") ? parseDate(json.get("query_last"), now) : new Date();
        this.retrieval_last = json.has("retrieval_last") ? parseDate(json.get("retrieval_last"), now) : new Date();
        this.retrieval_next = json.has("retrieval_next") ? parseDate(json.get("retrieval_next"), now) : new Date();
        this.expected_next = json.has("expected_next") ? parseDate(json.get("expected_next"), now) : new Date();
        this.query_count = (int) parseLong((Number) json.get("query_count"));
        this.retrieval_count = (int) parseLong((Number) json.get("retrieval_count"));
        this.message_period = parseLong((Number) json.get("message_period"));
        this.messages_per_day = (int) parseLong((Number) json.get("messages_per_day"));
        this.score_retrieval = (int) parseLong((Number) json.get("score_retrieval"));
        this.score_suggest = (int) parseLong((Number) json.get("score_suggest"));
    }
    
    /**
     * update the query entry
     * @param message_period 
     * @param byUserQuery is true, if the query was submitted by the user; false if the query was submitted by an automatic system
     */
    public void update(final long message_period, final boolean byUserQuery) {
        // message_period may have the value Long.MAX_VALUE if search requests have been empty and a message period cannot be computed
        this.retrieval_last = new Date();
        this.retrieval_count++;
        if (byUserQuery) {
            this.query_count++;
            this.query_last = this.retrieval_last;
        }
        long new_message_period = message_period; // can be Long.MAX_VALUE if less than 2 messages are in timeline!
        int new_messages_per_day = (int) (DAY_MILLIS / (new_message_period + 1)); // this is an interpolation based on the last tweet list, can be 0!
        if (new_message_period == Long.MAX_VALUE || new_messages_per_day == 0) {
            this.message_period = DAY_MILLIS;
        } else {
            this.message_period = this.message_period == 0 ? new_message_period : (this.message_period + new_message_period) / 2;
        }
        this.messages_per_day = (int) (DAY_MILLIS / this.message_period);
        double ttl_factor = DAO.getConfig("retrieval.queries.ttlfactor", 0.75d);
        long pivot_period = DAO.getConfig("retrieval.queries.pivotfrequency", 10000);
        this.expected_next = new Date(this.retrieval_last.getTime() + ((long) (ttl_factor *  this.message_period)));
        long strategic_period =   // if the period is far below the minimum, we apply a penalty
                 (this.message_period < pivot_period ?
                     pivot_period + 1000 * (long) Math.pow((pivot_period - this.message_period) / 1000, 3) :
                this.message_period);
        long waitingtime = Math.min(DAY_MILLIS, (long) (ttl_factor * RETRIEVAL_CONSTANT * strategic_period));
        this.retrieval_next = new Date(this.retrieval_last.getTime() + waitingtime);
    }
    // to check the retrieval order created by the update method, call
    // http://localhost:9000/api/suggest.json?orderby=retrieval_next&order=asc
    
    /**
     * A 'blind' update can be done if the user submits a query but there are rules which prevent that the target system is queried
     * as well. Then the query result is calculated using the already stored messages. To reflect this, only the query-related
     * attributes are changed.
     */
    public void update() {
        this.query_count++;
        this.query_last = new Date();
    }
    
    public String getQuery() {
        return this.query;
    }

    public int getQueryLength() {
        return this.query_length;
    }

    public SourceType getSourceType() {
        return this.source_type;
    }

    public Date getQueryFirst() {
        return this.query_first;
    }

    public Date getQueryLast() {
        return this.query_last;
    }

    public Date getRetrievalLast() {
        return this.retrieval_last;
    }

    public Date getRetrievalNext() {
        return this.retrieval_next;
    }

    public Date getExpectedNext() {
        return this.expected_next;
    }

    public int getTimezoneOffset() {
        return this.timezoneOffset;
    }

    public int getQueryCount() {
        return this.query_count;
    }

    public int getRetrievalCount() {
        return this.retrieval_count;
    }
    
    public long getMessagePeriod() {
        return this.message_period;
    }

    public int getMessagesPerDay() {
        return this.messages_per_day;
    }

    @Override
    public JSONObject toJSON() {
        JSONObject m = new JSONObject();
        m.put("query", this.query);
        m.put("query_length", this.query_length);
        m.put("source_type", this.source_type.toString());
        m.put("timezoneOffset", this.timezoneOffset);
        if (this.query_first != null) m.put("query_first", utcFormatter.print(this.query_first.getTime()));
        if (this.query_last != null) m.put("query_last", utcFormatter.print(this.query_last.getTime()));
        if (this.retrieval_last != null) m.put("retrieval_last", utcFormatter.print(this.retrieval_last.getTime()));
        if (this.retrieval_next != null) m.put("retrieval_next", utcFormatter.print(this.retrieval_next.getTime()));
        if (this.expected_next != null) m.put("expected_next", utcFormatter.print(this.expected_next.getTime()));
        m.put("query_count", this.query_count);
        m.put("retrieval_count", this.retrieval_count);
        m.put("message_period", this.message_period);
        m.put("messages_per_day", this.messages_per_day);
        m.put("score_retrieval", this.score_retrieval);
        m.put("score_suggest", this.score_suggest);
        return m;
    }

    private final static Pattern tokenizerPattern = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); // tokenizes Strings into terms respecting quoted parts
    
    private static enum Constraint {
        image("images"),
        audio("audio"),
        video("videos"),
        place("place_name"),
        location("location_point"),
        link("links"),
        mention("mentions"),
        source_type("source_type"),
        hashtag("hashtags"),
        emotion("classifier_emotion"),
        profanity("classifier_profanity"),
        language("classifier_language"),
        pure(""),
        len25("text_length"), len50("text_length"), len75("text_length"), len100("text_length");
        protected String field_name;
        protected Pattern pattern;
        private Constraint(String field_name) {
            this.field_name = field_name;
            this.pattern = Pattern.compile("\\s?\\-?/" + this.name() + "\\S*");
        }
    }
    
    public static class Tokens {
        
        public final String original;
        public String raw;
        public final HashSet<String> constraints_positive, constraints_negative;
        public Multimap<String, String> modifier;
        public PlaceContext place_context;
        public double[] bbox; // double[]{lon_west,lat_south,lon_east,lat_north}
        
        public Tokens(final String q) {
            this.original = q;
            List<String> tokens = new ArrayList<String>();
            Matcher m = tokenizerPattern.matcher(q);
            while (m.find()) tokens.add(m.group(1));

            this.constraints_positive = new HashSet<>();
            this.constraints_negative = new HashSet<>();
            this.modifier = HashMultimap.create();
            StringBuilder rawb = new StringBuilder(q.length() + 1);
            Set<String> hashtags = new HashSet<>();
            for (String t: tokens) {
                if (t.startsWith("/")) {
                    constraints_positive.add(t.substring(1));
                    continue;
                } else if (t.startsWith("-/")) {
                    constraints_negative.add(t.substring(2));
                    continue;
                } else if (t.indexOf(':') > 0) {
                    int p = t.indexOf(':');
                    modifier.put(t.substring(0, p).toLowerCase(), t.substring(p + 1));
                    rawb.append(t).append(' ');
                    continue;
                } else {
                    if (t.startsWith("#")) hashtags.add(t.substring(1));
                    rawb.append(t).append(' ');
                }
            }
            this.place_context = this.constraints_positive.remove("about") ? PlaceContext.ABOUT : PlaceContext.FROM;
            if (this.constraints_negative.remove("about")) this.place_context = PlaceContext.FROM;
            if (rawb.length() > 0 && rawb.charAt(rawb.length() - 1) == ' ') rawb.setLength(rawb.length() - 1);
            this.raw = rawb.toString();
            // fix common mistake using hashtags in combination with their words without hashtag
            for (String h: hashtags) {
                int p = this.raw.indexOf(h + " #" + h);
                if (p >= 0) this.raw = this.raw.substring(0,  p) + h + " OR #" + h + this.raw.substring(p + h.length() * 2 + 2);
                p = this.raw.indexOf("#" + h + " " + h);
                if (p >= 0) this.raw = this.raw.substring(0,  p) + "#" + h + " OR " + h + this.raw.substring(p + h.length() * 2 + 2);
            }
            
            // find bbox
            this.bbox = null;
            bboxsearch: for (String cs: this.constraints_positive) {
                if (cs.startsWith(Constraint.location.name() + "=")) {
                    String params = cs.substring(Constraint.location.name().length() + 1);
                    String[] coord = params.split(",");
                    if (coord.length == 4) {
                        this.bbox = new double[4];
                        for (int i = 0; i < 4; i++) this.bbox[i] = Double.parseDouble(coord[i]);
                        break bboxsearch;
                    }
                }
            }
            
            if (modifier.containsKey("near")) {
                // either check coordinates or name
                String near_name = modifier.get("near").iterator().next();
                GeoMark loc = DAO.geoNames.analyse(near_name, null, 10, Long.toString(System.currentTimeMillis()));
                if (loc != null) {
                    this.bbox = new double[]{loc.lon() - 1.0, loc.lat() + 1.0, loc.lon() + 1.0, loc.lat() - 1.0};
                }
            }
        }
        
        public String translate4scraper() {
            // check if a location constraint was given
            String dateclean = this.raw.replace(" since:hour", "").replace(" since:day", "").replace(" since:week", "").replace(" since:all", "").replace(" since:alltime", "").replace(" since:wholetime", "");
            if (this.bbox == null || this.original.indexOf("near:") > 0) {
                return dateclean;
            }
            // find place within the bbox
            double lon_west  = this.bbox[0];
            double lat_south = this.bbox[1];
            double lon_east  = this.bbox[2];
            double lat_north = this.bbox[3];
            assert lon_west < lon_east;
            assert lat_north > lat_south;
            // find largest city around to compute a 'near:' operator for twitter
            double lon_km = 40000 / 360 * (lon_east - lon_west);
            double lat_km = 40000 / 360 * (lat_north- lat_south);
            double within_km = Math.max(2.0, Math.max(lon_km, lat_km) / 2);
            double lon_border = (lon_east - lon_west) / 3;
            double lat_border = (lat_north - lat_south) / 3;
            GeoLocation largestCity = DAO.geoNames.getLargestCity(lon_west + lon_border, lat_south + lat_border, lon_east - lon_border, lat_north - lat_border);
            if (largestCity == null) largestCity = DAO.geoNames.getLargestCity(lon_west, lat_south, lon_east, lat_north);
            if (largestCity == null) largestCity = DAO.geoNames.cityNear((lat_north + lat_south) / 2.0, (lon_east + lon_west) / 2.0);
            String q = dateclean + " near:\"" + largestCity.getNames().iterator().next() + "\" within:" + ((int) (within_km / 1.609344)) + "mi"; // stupid imperial units are stupid
            return q;
        }
    }
    
    public static TwitterTimeline applyConstraint(TwitterTimeline tl0, Tokens tokens, boolean applyLocationConstraint) {
         if (tokens.constraints_positive.size() == 0 && tokens.constraints_negative.size() == 0 && tokens.modifier.size() == 0) return tl0;
        TwitterTimeline tl1 = new TwitterTimeline(tl0.getOrder());
        messageloop: for (TwitterTweet message: tl0) {

            // check modifier
            if (tokens.modifier.containsKey("from")) {
                for (String screen_name: tokens.modifier.get("from")) {
                    if (!message.getScreenName().equals(screen_name)) continue messageloop;
                }
            }
            if (tokens.modifier.containsKey("-from")) {
                for (String screen_name: tokens.modifier.get("-from")) {
                    if (message.getScreenName().equals(screen_name)) continue messageloop;
                }
            }
            if (applyLocationConstraint && tokens.bbox != null) {
                double[] location_point = message.getLocationPoint();
                if (location_point == null || location_point.length < 2) continue messageloop; //longitude, latitude
                if (location_point[0] < tokens.bbox[0] || location_point[0] > tokens.bbox[2] ||  // double[]{lon_west,lat_south,lon_east,lat_north}
                    location_point[1] > tokens.bbox[1] || location_point[1] < tokens.bbox[3]) continue messageloop;
            }
            
            // check constraints
            if (tokens.constraints_positive.contains("pure") && (
                    message.getImages().size() != 0 ||
                    message.getMentions().length != 0 ||
                    message.getLinks().length != 0 ||
                    message.getHashtags().length != 0
               )) continue;
            
            if (tokens.constraints_positive.contains("len25") && message.getTextLength() > 25) continue;
            if (tokens.constraints_positive.contains("len50") && message.getTextLength() > 50) continue;
            if (tokens.constraints_positive.contains("len75") && message.getTextLength() > 75) continue;
            if (tokens.constraints_positive.contains("len100") && message.getTextLength() > 100) continue;
            
            if (tokens.constraints_positive.contains("image") && message.getImages().size() == 0) continue;
            if (tokens.constraints_negative.contains("image") && message.getImages().size() != 0) continue;
            if (tokens.constraints_positive.contains("place") && message.getPlaceName().length() == 0) continue;
            if (tokens.constraints_negative.contains("place") && message.getPlaceName().length() != 0) continue;
            if (tokens.constraints_positive.contains("location") && (message.getLocationPoint() == null || message.getPlaceContext() != tokens.place_context)) continue;
            if (tokens.constraints_negative.contains("location") && message.getLocationPoint() != null) continue;
            if (tokens.constraints_positive.contains("link") && message.getLinks().length == 0) continue;
            if (tokens.constraints_negative.contains("link") && message.getLinks().length != 0) continue;
            if (tokens.constraints_positive.contains("mention") && message.getMentions().length == 0) continue;
            if (tokens.constraints_negative.contains("mention") && message.getMentions().length != 0) continue;
            if (tokens.constraints_positive.contains("hashtag") && message.getHashtags().length == 0) continue;
            if (tokens.constraints_negative.contains("hashtag") && message.getHashtags().length != 0) continue;
            for (Classifier.Context context: Classifier.Context.values()) {
                if (tokens.constraints_positive.contains(context.name()) && message.getClassifier(context) == null) continue messageloop;
                if (tokens.constraints_negative.contains(context.name()) && message.getClassifier(context) != null) continue messageloop;
            }
            
            // special treatment of location and link constraint
            constraintCheck: for (String cs: tokens.constraints_positive) {
                if (cs.startsWith(Constraint.location.name() + "=")) {
                    if (message.getLocationPoint() == null) continue messageloop;
                    if (message.getPlaceContext() != tokens.place_context) continue messageloop;
                    String params = cs.substring(Constraint.location.name().length() + 1);
                    String[] coord = params.split(",");
                    if (coord.length == 4) {
                        double lon = message.getLocationPoint()[0];
                        double lon_west  = Double.parseDouble(coord[0]);
                        double lon_east  = Double.parseDouble(coord[2]);
                        assert lon_west < lon_east;
                        if (lon < lon_west || lon > lon_east) continue messageloop;
                        double lat = message.getLocationPoint()[1];
                        double lat_south = Double.parseDouble(coord[1]);
                        double lat_north = Double.parseDouble(coord[3]);
                        assert lat_north > lat_south;
                        if (lat < lat_south || lat > lat_north) continue messageloop;
                    }
                }
                if (cs.startsWith(Constraint.link.name() + "=")) {
                    if (message.getLinks().length == 0) continue messageloop;
                    Pattern regex = Pattern.compile(cs.substring(Constraint.link.name().length() + 1));
                    for (String link: message.getLinks()) {
                        if (regex.matcher(link).matches()) continue constraintCheck;
                    }
                    // no match
                    continue messageloop;
                }
            }
            
            tl1.add(message, tl0.getUser(message.getScreenName()));
        }
        return tl1;
    }
    
    private final static Pattern term4ORPattern = Pattern.compile("(?:^| )(\\S*(?: OR \\S*)+)(?: |$)"); // Pattern.compile("(^\\s*(?: OR ^\\s*+)+)");
    
    private static List<String> splitIntoORGroups(String q) {
        // detect usage of OR junctor usage. Right now we cannot have mixed AND and OR usage. Thats a hack right now
        q = q.replaceAll(" AND ", " "); // AND is default
        
        // tokenize the query
        ArrayList<String> list = new ArrayList<>();
        Matcher m = term4ORPattern.matcher(q);
        while (m.find()) {
            String d = m.group(1);
            q = q.replace(d, "").replace("  ", " ");
            list.add(d);
            m = term4ORPattern.matcher(q);
        }
        q = q.trim();
        if (q.length() > 0) list.add(0, q);
        return list;
    }
    
    public static void main(String[] args) {
        splitIntoORGroups("Alpha OR Beta AND Gamma /constraint sand OR kies OR wasser skilanglauf");
    }
    
    /**
     * fixing a query mistake covers most common wrong queries from the user
     * @param q
     * @return the fixed query
     */
    public static String fixQueryMistakes(String q) {
        q = q.replaceAll("hashtag:", "#");
        q = q.replaceAll(" AND ", " "); // AND is default
        return q;
    }
    
    public static class ElasticsearchQuery {
        
        public QueryBuilder queryBuilder;
        public Date since;
        public Date until;

        public ElasticsearchQuery(String q, int timezoneOffset, Set<String> filterList) {
            // default values for since and util
            this.since = new Date(0);
            this.until = new Date(Long.MAX_VALUE);

            // parse the query
            this.queryBuilder = preparse(q, timezoneOffset, filterList);
        }

        public ElasticsearchQuery(
                Map<String, String> getMap,
                Map<String, String> notGetMap,
                Map<String, String> mayAlsoGetMap) {
            // default values for since and util
            this.since = new Date(0);
            this.until = new Date(Long.MAX_VALUE);

            // parse the query
            this.queryBuilder = preparse(getMap, notGetMap, mayAlsoGetMap);
        }

        public ElasticsearchQuery(String q, int timezoneOffset) {
            this(q, timezoneOffset, new HashSet<>());
        }

        private QueryBuilder preparse(String q, int timezoneOffset, Set<String> filterList) {
            // detect usage of OR connector usage.
            q = QueryEntry.fixQueryMistakes(q);
            List<String> terms = splitIntoORGroups(q); // OR binds stronger than AND
            if (terms.size() == 0) return QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery());
            // special handling
            if (terms.size() == 1) return parse(terms.get(0), timezoneOffset, filterList);

            // generic handling
            BoolQueryBuilder aquery = QueryBuilders.boolQuery();
            for (String t: terms) {
                QueryBuilder partial = parse(t, timezoneOffset, filterList);
                aquery.filter(partial);
            }
            return aquery;
        }

        private QueryBuilder preparse(
                Map<String, String> getMap,
                Map<String, String> notGetMap,
                Map<String, String> mayAlsoGetMap) {
            BoolQueryBuilder query = new BoolQueryBuilder();
            // Result must have these fields. Acts as AND operator
            if(getMap != null) {
                for(Map.Entry<String, String> field : getMap.entrySet()) {
                    query.must(QueryBuilders.termQuery(field.getKey(), field.getValue()));
                }
            }
            // Result must have these fields.
            if(notGetMap != null) {
                for(Map.Entry<String, String> field : notGetMap.entrySet()) {
                    query.mustNot(QueryBuilders.termQuery(field.getKey(), field.getValue()));
                }
            }
            // Result may preferably also get these fields. Acts as OR operator
            if(mayAlsoGetMap != null) {
                for(Map.Entry<String, String> field : mayAlsoGetMap.entrySet()) {
                    query.should(QueryBuilders.termQuery(field.getKey(), field.getValue()));
                }
            }
            return query;
        }

        private QueryBuilder preparse(String q, int timezoneOffset) {
            return preparse(q, timezoneOffset, new HashSet<>());
        }

        private QueryBuilder parse (
                String q,
                int timezoneOffset, 
                Set<String> filterList
        ) {
            // detect usage of OR ORconnective usage. Because of the preparse step we will have only OR or only AND here.
            q = q.replaceAll(" AND ", " "); // AND is default
            boolean ORconnective = q.indexOf(" OR ") >= 0;
            q = q.replaceAll(" OR ", " "); // if we know that all terms are OR, we remove that and apply it later

            // tokenize the query
            Set<String> qe = new LinkedHashSet<String>();
            Matcher m = tokenizerPattern.matcher(q);
            while (m.find()) qe.add(m.group(1));
            
            // twitter search syntax:
            //   term1 term2 term3 - all three terms shall appear
            //   "term1 term2 term3" - exact match of all terms
            //   term1 OR term2 OR term3 - any of the three terms shall appear
            //   from:user - tweets posted from that user
            //   to:user - tweets posted to that user
            //   @user - tweets which mention that user
            //   near:"location" within:xmi - tweets that are near that location
            //   #hashtag - tweets containing the given hashtag
            //   since:2015-04-01 until:2015-04-03 - tweets within given time range
            // additional constraints:
            //   /image /audio /video /place - restrict to tweets which have attached images, audio, video or place
            ArrayList<String> text_positive_match = new ArrayList<>();
            ArrayList<String> text_negative_match = new ArrayList<>();
            ArrayList<String> text_positive_filter = new ArrayList<>();
            ArrayList<String> text_negative_filter = new ArrayList<>();
            ArrayList<String> users_positive = new ArrayList<>();
            ArrayList<String> users_negative = new ArrayList<>();
            ArrayList<String> hashtags_positive = new ArrayList<>();
            ArrayList<String> hashtags_negative = new ArrayList<>();
            Multimap<String, String> modifier = HashMultimap.create();
            Set<String> constraints_positive = new HashSet<>();
            Set<String> constraints_negative = new HashSet<>();
            for (String t: qe) {
                if (t.length() == 0) continue;
                if (t.startsWith("@")) {
                    users_positive.add(t.substring(1));
                    continue;
                } else if (t.startsWith("-@")) {
                    users_negative.add(t.substring(2));
                    continue;
                } else if (t.startsWith("#")) {
                    hashtags_positive.add(t.substring(1));
                    continue;
                } else if (t.startsWith("-#")) {
                    hashtags_negative.add(t.substring(2));
                    continue;
                } else if (t.startsWith("/")) {
                    constraints_positive.add(t.substring(1));
                    continue;
                } else if (t.startsWith("-/")) {
                    constraints_negative.add(t.substring(2));
                    continue;
                } else if (t.indexOf(':') > 0) {
                    int p = t.indexOf(':');
                    modifier.put(t.substring(0, p).toLowerCase(), t.substring(p + 1));
                    continue;
                } else {
                    // patch characters that will confuse elasticsearch or have a different meaning
                    boolean negative = t.startsWith("-");
                    if (negative) t = t.substring(1);
                    if (t.length() == 0) continue;
                    if ((t.charAt(0) == '"' && t.charAt(t.length() - 1) == '"') || (t.charAt(0) == '\'' && t.charAt(t.length() - 1) == '\'')) {
                        t = t.substring(1, t.length() - 1);
                        if (negative) text_negative_filter.add(t); else text_positive_filter.add(t);
                    } else if (t.indexOf('-') > 0) {
                        // this must be handled like a quoted string without the minus
                        t = t.replaceAll("-", " ");
                        if (negative) text_negative_filter.add(t); else text_positive_filter.add(t);
                    } else {
                        if (negative) text_negative_match.add(t); else text_positive_match.add(t);
                    }
                    continue;
                }
            }
            if (modifier.containsKey("to")) users_positive.addAll(modifier.get("to"));
            if (modifier.containsKey("-to")) users_negative.addAll(modifier.get("-to"));

            // special constraints
            boolean constraint_about = constraints_positive.remove("about");
            if (constraints_negative.remove("about")) constraint_about = false;
            
            // compose query for text
            List<QueryBuilder> ops = new ArrayList<>();
            List<QueryBuilder> nops = new ArrayList<>();
            List<QueryBuilder> filters = new ArrayList<>();
            if (text_positive_match.size() == 1) {
                BoolQueryBuilder disjunction = QueryBuilders.boolQuery();
                disjunction.should(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", text_positive_match.get(0))));
                disjunction.should(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("text", text_positive_match.get(0))));
                disjunction.minimumNumberShouldMatch(1);
                ops.add(disjunction);
            } else {
                for (String text: text_positive_match)  {
                    ops.add(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("text", text)));
                }
            }
            if (text_negative_match.size() == 1) {
                BoolQueryBuilder disjunction = QueryBuilders.boolQuery();
                disjunction.should(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", text_negative_match.get(0))));
                disjunction.should(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("text", text_negative_match.get(0))));
                disjunction.minimumNumberShouldMatch(1);
                ops.add(disjunction);
            } else {
                for (String text: text_negative_match) {
                    // negation of terms in disjunctions would cause to retrieve almost all documents
                    // this cannot be the requirement of the user. It may be valid in conjunctions, but not in disjunctions
                    nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("text", text)));
                }
            }
            
            // apply modifiers
            if (modifier.containsKey("id")) {
                ops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("id_str", modifier.get("id"))));
            }
            if (modifier.containsKey("-id")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("id_str", modifier.get("-id"))));
            }

            for (String user: users_positive) {
                ops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("mentions", user)));
            }
            for (String user: users_negative) nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("mentions", user)));
            
            for (String hashtag: hashtags_positive) {
                ops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("hashtags", hashtag.toLowerCase())));
            }
            for (String hashtag: hashtags_negative) nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("hashtags", hashtag.toLowerCase())));

            if (constraints_positive.remove("pure")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.image.field_name)));
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.audio.field_name)));
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.video.field_name)));
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.link.field_name)));
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.mention.field_name)));
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.existsQuery(Constraint.hashtag.field_name)));
            };
            if (constraints_positive.remove("len25")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders
                        .rangeQuery(Constraint.len25.field_name).from(26).to(1000).includeLower(true).includeUpper(false)));
            };
            if (constraints_positive.remove("len50")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders
                        .rangeQuery(Constraint.len50.field_name).from(51).to(1000).includeLower(true).includeUpper(false)));
            };
            if (constraints_positive.remove("len75")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders
                        .rangeQuery(Constraint.len75.field_name).from(76).to(1000).includeLower(true).includeUpper(false)));
            };
            if (constraints_positive.remove("len100")) {
                nops.add(QueryBuilders.constantScoreQuery(QueryBuilders
                        .rangeQuery(Constraint.len100.field_name).from(101).to(1000).includeLower(true).includeUpper(false)));
            };
            if (modifier.containsKey("from")) {
                for (String screen_name: modifier.get("from")) {
                    if (screen_name.indexOf(',') < 0) {
                        ops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", screen_name)));
                    } else {
                        String[] screen_names = screen_name.split(",");
                        BoolQueryBuilder disjunction = QueryBuilders.boolQuery();
                        for (String name: screen_names) disjunction.should(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", name)));
                        disjunction.minimumNumberShouldMatch(1);
                        ops.add(disjunction);
                    }
                }
            }
            if (modifier.containsKey("-from")) {
                for (String screen_name: modifier.get("-from")) {
                    if (screen_name.indexOf(',') < 0) {
                        nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", screen_name)));
                    } else {
                        String[] screen_names = screen_name.split(",");
                        for (String name: screen_names) nops.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("screen_name", name)));
                    }
                }
            }
            if (modifier.containsKey("near")) {
                // either check coordinates or name
                String near_name = modifier.get("near").iterator().next();
                GeoMark loc = DAO.geoNames.analyse(near_name, null, 10, Long.toString(System.currentTimeMillis()));
                if (loc == null) {
                    BoolQueryBuilder nearquery = QueryBuilders.boolQuery()
                        .should(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("place_name", near_name)))
                        .should(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("text", near_name)));
                    nearquery.minimumNumberShouldMatch(1);
                    ops.add(QueryBuilders.boolQuery().filter(nearquery).filter(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("place_context", PlaceContext.FROM.name()))));
                } else {                    
                    filters.add(QueryBuilders.geoDistanceQuery("location_point").distance(100.0, DistanceUnit.KILOMETERS).lat(loc.lat()).lon(loc.lon()));
                }
            }
            if (modifier.containsKey("since")) try {
                Calendar since = DateParser.parse(modifier.get("since").iterator().next(), timezoneOffset);
                this.since = since.getTime();
                RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(AbstractObjectEntry.CREATED_AT_FIELDNAME).from(this.since);
                if (modifier.containsKey("until")) {
                    Calendar until = DateParser.parse(modifier.get("until").iterator().next(), timezoneOffset);
                    if (until.get(Calendar.HOUR) == 0 && until.get(Calendar.MINUTE) == 0) {
                        // until must be the day which is included in results.
                        // To get the result within the same day, we must add one day.
                        until.add(Calendar.DATE, 1);
                    }
                    this.until = until.getTime();
                    rangeQuery.to(this.until);
                } else {
                    this.until = new Date(Long.MAX_VALUE);
                }
                ops.add(rangeQuery);
            } catch (ParseException e) {} else if (modifier.containsKey("until")) try {
                Calendar until = DateParser.parse(modifier.get("until").iterator().next(), timezoneOffset);
                if (until.get(Calendar.HOUR) == 0 && until.get(Calendar.MINUTE) == 0) {
                    // until must be the day which is included in results.
                    // To get the result within the same day, we must add one day.
                    until.add(Calendar.DATE, 1);
                }
                this.until = until.getTime();
                RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(AbstractObjectEntry.CREATED_AT_FIELDNAME).to(this.until);
                ops.add(rangeQuery);
            } catch (ParseException e) {}

            // apply the ops and nops
            QueryBuilder bquery = QueryBuilders.boolQuery();
            if (ops.size() == 1 && nops.size() == 0)
                bquery = ops.iterator().next();
            else if (ops.size() == 0 && nops.size() == 1)
                bquery = QueryBuilders.boolQuery().mustNot(ops.iterator().next());
            else {
                for (QueryBuilder qb: ops) {
                    if (ORconnective) ((BoolQueryBuilder) bquery).should(qb); else ((BoolQueryBuilder) bquery).filter(qb);
                }
                if (ORconnective) ((BoolQueryBuilder) bquery).minimumNumberShouldMatch(1);
                for (QueryBuilder nqb: nops) {
                    ((BoolQueryBuilder) bquery).mustNot(nqb);
                }
                
            }
            
            // apply constraints as filters
            for (String text: text_positive_filter) {
                filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("text", text)));
            }
            for (String text: text_negative_filter) filters.add(QueryBuilders.boolQuery().mustNot(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("text", text))));
            for (Constraint c: Constraint.values()) {
                if (c.field_name != null && c.field_name.length() > 0) {
                    if (constraints_positive.contains(c.name())) {
                        filters.add(QueryBuilders.existsQuery(c.field_name));
                    }
                    if (constraints_negative.contains(c.name())) {
                        filters.add(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(c.field_name)));
                    }
                }
            }
            if (constraints_positive.contains("location")) {
                filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("place_context", (constraint_about ? PlaceContext.ABOUT : PlaceContext.FROM).name())));
            }

            if (filterList.size() > 0) {
                
                for (String filter: filterList) {
                    switch(filter) {
                        case "image":
                        case "video":
                            // filter result if images_count (or video_count) is 0
                            filters.add(QueryBuilders.boolQuery().mustNot(
                                    QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(filter + "s_count", "0"))
                            ));
                            break;
                        // TODO: Add more filters here

                        default:
                            break;
                    } 
                }
            }
            // special treatment of location constraints of the form /location=lon-west,lat-south,lon-east,lat-north i.e. /location=8.58,50.178,8.59,50.181
            //                      source_type constraint of the form /source_type=FOSSASIA_API -> search exact term (source_type must exists in SourceType enum)
            for (String cs: constraints_positive) {
                if (cs.startsWith(Constraint.location.name() + "=")) {
                    String params = cs.substring(Constraint.location.name().length() + 1);
                    String[] coord = params.split(",");
                    if (coord.length == 1) {
                        filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("location_source", coord[0])));
                    } else if (coord.length == 2) {
                        double lon = Double.parseDouble(coord[0]);
                        double lat = Double.parseDouble(coord[1]);
                        // ugly way to search exact geo_point : using geoboundingboxfilter, with two identical bounding points
                        // geoshape filter can search for exact point shape but it can't be performed on geo_point field
                        filters.add(QueryBuilders.geoBoundingBoxQuery("location_point")
                                .topLeft(lat, lon)
                                .bottomRight(lat, lon));
                    }
                    else if (coord.length == 4 || coord.length == 5) {
                        double lon_west  = Double.parseDouble(coord[0]);
                        double lat_south = Double.parseDouble(coord[1]);
                        double lon_east  = Double.parseDouble(coord[2]);
                        double lat_north = Double.parseDouble(coord[3]);
                        PlaceContext context = constraint_about ? PlaceContext.ABOUT : PlaceContext.FROM;
                        filters.add(QueryBuilders.existsQuery(Constraint.location.field_name));
                        filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("place_context", context.name())));
                        filters.add(QueryBuilders.geoBoundingBoxQuery("location_point")
                                .topLeft(lat_north, lon_west)
                                .bottomRight(lat_south, lon_east));
                        if (coord.length == 5) filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery("location_source", coord[4])));
                    }
                } else if (cs.startsWith(Constraint.link.name() + "=")) {
                    String regexp = cs.substring(Constraint.link.name().length() + 1);
                    filters.add(QueryBuilders.existsQuery(Constraint.link.field_name));
                    filters.add(QueryBuilders.regexpQuery(Constraint.link.field_name, regexp));
                } else if (cs.startsWith(Constraint.source_type.name() + "=")) {
                    String regexp = cs.substring(Constraint.source_type.name().length() + 1);
                    if (SourceType.isValid(regexp)) {
                        filters.add(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("_type", regexp)));
                    }
                }
            }

            for (String cs : constraints_negative) {
                if (cs.startsWith(Constraint.source_type.name() + "=")) {
                    String regexp = cs.substring(Constraint.source_type.name().length() + 1);
                    if (SourceType.isValid(regexp)) {
                        filters.add(QueryBuilders.boolQuery().mustNot(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("_type", regexp))));
                    }
                }
            }

            BoolQueryBuilder queryFilter = QueryBuilders.boolQuery();
            for(QueryBuilder filter : filters){
            	queryFilter.filter(filter);
            }
            QueryBuilder cquery = filters.size() == 0 ? bquery : QueryBuilders.boolQuery().filter(bquery).filter(queryFilter);
            return cquery;
        }

        private QueryBuilder parse (String q, int timezoneOffset) {        
            return parse(q, timezoneOffset, new HashSet<>());
        }


    }

    public static enum PlaceContext {
        
        FROM,  // the message was made at that place
        ABOUT; // the message is about that place
        
    }

} public class TwitterTimeline extends BasicTimeline<TwitterTweet> implements Iterable<TwitterTweet> {
    
    public TwitterTimeline(Order order) {
        super(order);
    }
    
    public TwitterTimeline(Order order, String scraperInfo) {
        super(order, scraperInfo);
    }
    
    public TwitterTimeline reduceToMaxsize(final int maxsize) {
        List<TwitterTweet> m = new ArrayList<>();
        TwitterTimeline t = new TwitterTimeline(this.order);
        if (maxsize < 0) return t;
        
        // remove tweets from this timeline
        synchronized (this.posts) {
            while (this.posts.size() > maxsize) m.add(this.posts.remove(this.posts.firstEntry().getKey()));
        }
        
        // create new timeline
        for (TwitterTweet me: m) {
            t.addUser(this.users.get(me.getScreenName()));
            t.addTweet(me);
        }
        
        // prune away users not needed any more in this structure
        Set<String> screen_names = new HashSet<>();
        for (TwitterTweet me: this.posts.values()) screen_names.add(me.getScreenName());
        synchronized (this.users) {
            Iterator<Map.Entry<String, UserEntry>> i = this.users.entrySet().iterator();
            while (i.hasNext()) {
                Map.Entry<String, UserEntry> e = i.next();
                if (!screen_names.contains(e.getValue().getScreenName())) i.remove();
            }
        }        
        return t;
    }
    
    public TwitterTimeline add(TwitterTweet tweet, UserEntry user) {
        this.addUser(user);
        this.addTweet(tweet);
        return this;
    }
    
    private TwitterTimeline addTweet(TwitterTweet tweet) {
        String key = "";
        if (this.order == Order.RETWEET_COUNT) {
            key = Long.toHexString(tweet.getRetweetCount());
            while (key.length() < 16) key = "0" + key;
            key = key + "_" + tweet.getPostId();
        } else if (this.order == Order.FAVOURITES_COUNT) {
            key = Long.toHexString(tweet.getFavouritesCount());
            while (key.length() < 16) key = "0" + key;
            key = key + "_" + tweet.getPostId();
        } else {
            key = Long.toHexString(tweet.getCreatedAt().getTime()) + "_" + tweet.getPostId();
        }
        synchronized (this.posts) {
            TwitterTweet precursorTweet = getPrecursorTweet();
            if (precursorTweet != null && tweet.getCreatedAt().before(precursorTweet.getCreatedAt())) return this; // ignore this tweet in case it would change the list of shown tweets
            this.posts.put(key, tweet);
        }
        return this;
    }
    
    public void putAll(TwitterTimeline other) {
        if (other == null) return;
        assert this.order.equals(other.order);
        for (Map.Entry<String, UserEntry> u: other.users.entrySet()) {
            UserEntry t = this.users.get(u.getKey());
            if (t == null || !t.containsProfileImage()) {
                this.users.put(u.getKey(), u.getValue());
            }
        }
        for (TwitterTweet t: other) this.addTweet(t);
    }
    
    public String toString() {
        return toJSON(true, "search_metadata", "statuses").toString();
        //return new ObjectMapper().writer().writeValueAsString(toMap(true));
    }

    public JSONObject toJSON(boolean withEnrichedData, String metadata_field_name, String statuses_field_name) throws JSONException {
        JSONObject json = toSusi(withEnrichedData, new SusiThought(metadata_field_name, statuses_field_name));
        json.getJSONObject(metadata_field_name).put("count", Integer.toString(this.posts.size()));
        json.put("peer_hash", DAO.public_settings.getPeerHash());
        json.put("peer_hash_algorithm", DAO.public_settings.getPeerHashAlgorithm());
        return json;
    }
    
    private SusiThought toSusi(boolean withEnrichedData, SusiThought json) throws JSONException {
        json
            .setQuery(this.query)
            .setHits(Math.max(this.hits, this.size()));
        if (this.scraperInfo.length() > 0) json.setScraperInfo(this.scraperInfo);
        JSONArray statuses = new JSONArray();
        for (TwitterTweet t: this) {
            UserEntry u = this.users.get(t.getScreenName());
            statuses.put(t.toJSON(u, withEnrichedData, Integer.MAX_VALUE, ""));
        }
        json.setData(statuses);
        return json;
    }

    /**
     * compute the average time between any two consecutive tweets
     * @return time in milliseconds
     */
    public long period() {
        if (this.size() < 1) return Long.MAX_VALUE;
        
        // calculate the time based on the latest 20 tweets (or less)
        long latest = 0;
        long earliest = 0;
        int count = 0;
        for (TwitterTweet TwitterTweet: this) {
            if (latest == 0) {latest = TwitterTweet.created_at.getTime(); continue;}
            earliest = TwitterTweet.created_at.getTime();
            count++;
            if (count >= 19) break;
        }

        if (count == 0) return Long.MAX_VALUE;
        long timeInterval = latest - earliest;
        long p = 1 + timeInterval / count;
        return p < 4000 ? p / 4 + 3000 : p;
    }    
    
    public void writeToIndex() {
        IncomingMessageBuffer.addScheduler(this, true, true);
    }

    //TODO: temporary method to prevent issues related to Timeline class popping-up till next PR
    public PostTimeline toPostTimeline() {
        PostTimeline postList = new PostTimeline(Order.TIMESTAMP);
        for (TwitterTweet me : this) {
            assert me instanceof Post;
            postList.add(me);
        }
        return postList;
    }

} public final class IO {
	
	private static Map<Path,String> map;
	private static boolean initialized = false;

	public static String readFile(@Nonnull Path path) throws IOException
	{
		byte[] encoded = Files.readAllBytes(path);
		return new String(encoded);
	}
	
	public static String readFileCached(@Nonnull Path path) throws IOException
	{
		Path absPath = path.toAbsolutePath();
		if(!initialized) init();
		if(map.containsKey(absPath)){
			return map.get(absPath);
		}
		else{
			String result = readFile(absPath);
			map.put(absPath, result);
			return result;
		}
	}
	
	private static void init(){
		map = new HashMap<Path,String>();
		initialized = true;
	}

	/**
	 * Create hash for a key
	 * @param pubkey
	 * @param algorithm
	 * @return String hash
	 */
	public static String getKeyHash(@Nonnull PublicKey pubkey, @Nonnull String algorithm){
		try {
			MessageDigest md = MessageDigest.getInstance(algorithm);
			md.update(pubkey.getEncoded());
			return Base64.getEncoder().encodeToString(md.digest());
		} catch (NoSuchAlgorithmException e) {
			DAO.severe(e);
		}
		return null;
	}

	/**
	 * Create hash for a key, use default algorithm SHA-256
	 * @param pubkey
	 * @return String hash
	 */
	public static String getKeyHash(@Nonnull PublicKey pubkey){
		return getKeyHash(pubkey, "SHA-256");
	}

	/**
	 * Get String representation of a key
	 * @param key
	 * @return String representation of a key
	 */
	public static String getKeyAsString(@Nonnull Key key){
		return Base64.getEncoder().encodeToString(key.getEncoded());
	}

	/**
	 * Create PublicKey from String representation
	 * @param encodedKey
	 * @param algorithm
	 * @return PublicKey public_key
	 */
	public synchronized static PublicKey decodePublicKey(@Nonnull String encodedKey, @Nonnull String algorithm){
		try{
			X509EncodedKeySpec keySpec = new X509EncodedKeySpec(Base64.getDecoder().decode(encodedKey));
			PublicKey pub = KeyFactory.getInstance(algorithm).generatePublic(keySpec);
			return pub;
		}
		catch(NoSuchAlgorithmException | InvalidKeySpecException e){
			DAO.severe(e);
		}
		return null;
	}
	
	/**
	 * Resolves an untrusted user-specified path against the API's base directory.
	 * Paths that try to escape the base directory are rejected.
	 *
	 * @param baseDirPath  the absolute path of the base directory that all
	 * user-specified paths should be within
	 * @param userPath  the untrusted path provided by the API user, expected to be
	 * relative to {@code baseDirPath}
	 */
	public static Path resolvePath(final Path baseDirPath, final Path userPath) {
		if (!baseDirPath.isAbsolute()) {
			throw new IllegalArgumentException("Base path must be absolute");
		}

		if (userPath.isAbsolute()) {
			throw new IllegalArgumentException("User path must be relative");
		}

		// Join the two paths together, then normalize so that any ".." elements
		// in the userPath can remove parts of baseDirPath.
		// (e.g. "/foo/bar/baz" + "../attack" -> "/foo/bar/attack")
		final Path resolvedPath = baseDirPath.resolve(userPath).normalize();

		// Make sure the resulting path is still within the required directory.
		// (In the example above, "/foo/bar/attack" is not.)
		if (!resolvedPath.startsWith(baseDirPath)) {
			throw new IllegalArgumentException("User path escapes the base path");
		}

		return resolvedPath;
	}

	public static Path resolvePath(final Path baseDirPath, final String userPath) {
		return resolvePath(baseDirPath, Paths.get(userPath));
	}

	/**
	 * Checks each subsequent path to be strictly within the baseDirPath so that
	 * no path argument leads to directory traversal attack
	 *
	 * E.g. /models/ + req.model + '/' + req.lang + /images/ + req.image
	 * Should be checked for ('models', req.model, req.lang, 'images', req.image)
	 * that each subsequent element is within the previous and not breaking out by passing
	 * req.model => ..
	 * req.lang  => ..
	 * req.image => ../../private/data.json
	 *
	 * Since just checking the last argument isn't enough
	 *
	 * @param baseDirPath the absolute path of the base directory that all
	 * user-specified paths should be within
	 * @param paths the untrusted paths provided by the API user, expected to be
	 * relative to {@code baseDirPath}
	 */
	public static Path resolvePath(final Path baseDirPath, final String... paths) {
		Path resolved = baseDirPath;
		for (String path: paths) {
			resolved = resolvePath(resolved, path);
		}

		return resolved;
	}
}
import com.fasterxml.jackson.databind.JsonNode;
import com.github.fge.jackson.JsonLoader;
import com.google.common.base.Charsets;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.nio.file.Path;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONArray;
import org.loklak.Caretaker;
import org.loklak.api.search.SearchServlet;
import org.loklak.api.search.WordpressCrawlerService;
import org.loklak.geo.GeoNames;
import org.loklak.harvester.Post;
import org.loklak.harvester.TwitterScraper;
import org.loklak.harvester.YoutubeScraper;
import org.loklak.harvester.TwitterScraper.TwitterTweet;
import org.loklak.api.search.GithubProfileScraper;
import org.loklak.api.search.InstagramProfileScraper;
import org.loklak.api.search.QuoraProfileScraper;
import org.loklak.api.search.TweetScraper;
import org.loklak.harvester.BaseScraper;
import org.loklak.http.AccessTracker;
import org.loklak.http.ClientConnection;
import org.loklak.http.RemoteAccess;
import org.loklak.ir.AccountFactory;
import org.loklak.ir.BulkWriteResult;
import org.loklak.ir.ElasticsearchClient;
import org.loklak.ir.ImportProfileFactory;
import org.loklak.ir.MessageFactory;
import org.loklak.ir.QueryFactory;
import org.loklak.ir.UserFactory;
import org.loklak.objects.AbstractObjectEntry;
import org.loklak.objects.AccountEntry;
import org.loklak.objects.ImportProfileEntry;
import org.loklak.objects.Peers;
import org.loklak.objects.QueryEntry;
import org.loklak.objects.ResultList;
import org.loklak.objects.SourceType;
import org.loklak.objects.TwitterTimeline;
import org.loklak.objects.PostTimeline;
import org.loklak.objects.TimelineCache;
import org.loklak.objects.UserEntry;
import org.loklak.server.*;
import org.loklak.stream.MQTTPublisher;
import org.loklak.tools.DateParser;
import org.loklak.tools.IO;
import org.loklak.tools.OS;
import org.loklak.tools.storage.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
    /**
     * Search the local account cache using an elasticsearch query.
     * @param screen_name - the user id
     */
    public static AccountEntry searchLocalAccount(final String screen_name) {
        try {
            return accounts.read(screen_name);
        } catch (IOException e) {
        	DAO.severe(e);
            return null;
        }
    }

    /**
     * Search the local message cache using a elasticsearch query.
     * @param q - the query, can be empty for a matchall-query
     * @param resultCount - the number of messages in the result
     * @param sort_field - the field name to sort the result list, i.e. "query_first"
     * @param sort_order - the sort order (you want to use SortOrder.DESC here)
     */
    public static ResultList<QueryEntry> SearchLocalQueries(final String q, final int resultCount, final String sort_field, final String default_sort_type, final SortOrder sort_order, final Date since, final Date until, final String range_field) {
        ResultList<QueryEntry> queries = new ResultList<>();
        ResultList<Map<String, Object>> result = elasticsearch_client.fuzzyquery(IndexName.queries.name(), "query", q, resultCount, sort_field, default_sort_type, sort_order, since, until, range_field);
        queries.setHits(result.getHits());
        for (Map<String, Object> map: result) {
            QueryEntry qe = new QueryEntry(new JSONObject(map));
            // check a flag value for queries that probably never get new messages
            if (qe.getMessagePeriod() != QueryEntry.DAY_MILLIS) {
                queries.add(qe);
            }
        }
        return queries;
    }

    public static ImportProfileEntry SearchLocalImportProfiles(final String id) {
        try {
            return importProfiles.read(id);
        } catch (IOException e) {
        	DAO.severe(e);
            return null;
        }
    }

    public static Collection<ImportProfileEntry> SearchLocalImportProfilesWithConstraints(final Map<String, String> constraints, boolean latest) throws IOException {
        List<ImportProfileEntry> rawResults = new ArrayList<>();
        List<Map<String, Object>> result = elasticsearch_client.queryWithConstraints(IndexName.import_profiles.name(), "active_status", ImportProfileEntry.EntryStatus.ACTIVE.name().toLowerCase(), constraints, latest);
        for (Map<String, Object> map: result) {
            rawResults.add(new ImportProfileEntry(new JSONObject(map)));
        }

        if (!latest) {
            return rawResults;
        }

        // filter results to display only latest profiles
        Map<String, ImportProfileEntry> latests = new HashMap<>();
        for (ImportProfileEntry entry : rawResults) {
            String uniqueKey;
            if (entry.getImporter() != null) {
                uniqueKey = entry.getSourceUrl() + entry.getImporter();
            } else {
                uniqueKey = entry.getSourceUrl() + entry.getClientHost();
            }
            if (latests.containsKey(uniqueKey)) {
                if (entry.getLastModified().compareTo(latests.get(uniqueKey).getLastModified()) > 0) {
                    latests.put(uniqueKey, entry);
                }
            } else {
                latests.put(uniqueKey, entry);
            }
        }
        return latests.values();
    }

    public static TwitterTimeline scrapeTwitter(
            final Query post,
            final String q,
            final TwitterTimeline.Order order,
            final int timezoneOffset,
            boolean byUserQuery,
            long timeout,
            boolean recordQuery) {

        return scrapeTwitter(post, new HashSet<>(), q, order, timezoneOffset, byUserQuery, timeout, recordQuery);
    }

    /**
     * 
     * @param post
     * @param filterList empty or one/several of image, video
     * @param q
     * @param order
     * @param timezoneOffset
     * @param byUserQuery
     * @param timeout
     * @param recordQuery
     * @return
     */
    public static TwitterTimeline scrapeTwitter(
            final Query post,
            final Set<String> filterList,
            final String q,
            final TwitterTimeline.Order order,
            final int timezoneOffset,
            boolean byUserQuery,
            long timeout,
            boolean recordQuery) {
        // retrieve messages from remote server

        ArrayList<String> remote = DAO.getFrontPeers();
        TwitterTimeline tl;
        if (remote.size() > 0 && (peerLatency.get(remote.get(0)) == null || peerLatency.get(remote.get(0)).longValue() < 3000)) {
            long start = System.currentTimeMillis();
            tl = searchOnOtherPeers(remote, q, filterList, order, 100, timezoneOffset, "all", SearchServlet.frontpeer_hash, timeout); // all must be selected here to catch up missing tweets between intervals
            // at this point the remote list can be empty as a side-effect of the remote search attempt
            if (post != null && remote.size() > 0 && tl != null) post.recordEvent("remote_scraper_on_" + remote.get(0), System.currentTimeMillis() - start);
            if (tl == null || tl.size() == 0) {
                // maybe the remote server died, we try then ourself
                start = System.currentTimeMillis();

                tl = TwitterScraper.search(q, filterList, order, true, true, 400);