Hey guys, Finally i changed all my queries to constantscorequeries. It's way better, but still, certain pages take a lot of time running... I don't understand why, and i don't have anything in my ES logs...
Now the average time for search 20 users and their mentions/timeline + scoring them is about 4s (and almost 4s for the search). But when it takes time, it's still 60s for 1 page !!! I tried reading the explain data i can't get after the query but there's no response time. How can I find a way to understand why certain queries take so much time ? Thanks ! Le lundi 18 août 2014 12:29:10 UTC+2, Pierrick Boutruche a écrit : > > Hi everyone ! > > I'm currently working on a tool with *ES and Twitter Streaming API*, in > which I try to find interesting profiles on Twitter, based on what they > tweet, RT and which of their interactions are shared/RT. > > Anyway, I use ES to index and search among tweets. To do that, I get > Twitter stream data and put in a *single index users & tweets (2 types)*, > linked by the user id via un parent-child relation. Actually, I thought of > my indexing a lot and it is the best way to do it. > - I need to update very often users (because i score them and because they > update their profile quite often), so get the user nested in the tweet is > not an option (too many replicas) > - I could put user's tweets directly in the user object but I would have > huge objects and I don't really want that. > > I work on a SoYouStart Server, 4c/4t 3.2GHz, 32Go RAM, 4To HDD. > > My settings for the index are : > > settings = { > > "index" : { > > "number_of_replicas" : 0, > > "refresh_interval" : '10s', > > "routing.allocation.disable_allocation": False > > }, > > "analysis": { > > "analyzer": { > > "snowFrench":{ > > "type": "snowball", > > "language": "French" > > }, > > "snowEnglish":{ > > "type": "snowball", > > "language": "English" > > }, > > "snowGerman":{ > > "type": "snowball", > > "language": "German" > > }, > > "snowRussian":{ > > "type": "snowball", > > "language": "Russian" > > }, > > "snowSpanish":{ > > "type": "snowball", > > "language": "Spanish" > > }, > > "snowJapanese":{ > > "type": "snowball", > > "language": "Japanese" > > }, > > "edgeNGramAnalyzer":{ > > "tokenizer": "myEdgeNGram" > > }, > > "name_analyzer": { > > "tokenizer": "whitespace", > > "type": "custom", > > "filter": ["lowercase", "multi_words", "name_filter"] > > }, > > "city_analyzer" : { > > "type" : "snowball", > > "language" : "English" > > } > > }, > > "tokenizer" : { > > "myEdgeNGram" : { > > "type" : "edgeNGram", > > "min_gram" : 2, > > "max_gram" : 5 > > }, > > "name_tokenizer": { > > "type": "edgeNGram", > > "max_gram": 100, > > "min_gram": 4 > > } > > }, > > "filter": { > > "multi_words": { > > "type": "shingle", > > "min_shingle_size": 2, > > "max_shingle_size": 10 > > }, > > "name_filter": { > > "type": "edgeNGram", > > "max_gram": 100, > > "min_gram": 4 > > } > > } > > } > > } > > > And my mappings are : > > tweet_mapping = { > > "_all" : { > "enabled" : False > }, > "_ttl" : { > "enabled" : True, > "default" : "400d" > }, > "_parent" : { > "type" : 'user' > }, > "properties": { > "textfr": { > 'type': 'string', > '_analyzer': 'snowFrench', > 'copy_to': 'text' > }, > "texten": { > 'type': 'string', > '_analyzer': 'snowEnglish', > 'copy_to': 'text' > }, > "textde": { > 'type': 'string', > '_analyzer': 'snowGerman', > 'copy_to': 'text' > }, > "textja": { > 'type': 'string', > '_analyzer': 'snowJapanese', > 'copy_to': 'text' > }, > "textru": { > 'type': 'string', > '_analyzer': 'snowRussian', > 'copy_to': 'text' > }, > "textes": { > 'type': 'string', > '_analyzer': 'snowSpanish', > 'copy_to': 'text' > }, > "text": { > 'type': 'string', > 'null_value': '', > 'index': 'analyzed', > 'store': 'yes' > }, > "entities": { > 'type': 'object', > 'index': 'analyzed', > 'store': 'yes', > 'properties': { > "hashtags": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > "_analyzer": "edgeNGramAnalyzer" > }, > "mentions": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'long', > 'precision_step': 64 > } > } > }, > "lang": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'string' > }, > "created_at": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'date', > 'format' : 'dd-MM-YYYY HH:mm:ss' > } > } > } > user_mapping = { > "_all" : { > "enabled" : False > }, > "_ttl" : { > "enabled" : True, > "default" : "600d" > }, > "properties": { > "lang": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'string' > }, > "name": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > "_analyzer": "edgeNGramAnalyzer" > }, > "screen_name": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > "_analyzer": "edgeNGramAnalyzer" > }, > "descfr": { > 'type': 'string', > '_analyzer': 'snowFrench', > 'copy_to': 'description' > }, > "descen": { > 'type': 'string', > '_analyzer': 'snowEnglish', > 'copy_to': 'description' > }, > "descde": { > 'type': 'string', > '_analyzer': 'snowGerman', > 'copy_to': 'description' > }, > "descja": { > 'type': 'string', > '_analyzer': 'snowJapanese', > 'copy_to': 'description' > }, > "descru": { > 'type': 'string', > '_analyzer': 'snowRussian', > 'copy_to': 'description' > }, > "desces": { > 'type': 'string', > '_analyzer': 'snowSpanish', > 'copy_to': 'description' > }, > "description": { > 'type': 'string', > 'null_value': '', > 'index': 'analyzed', > 'store': 'yes' > }, > "created_at": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'date', > 'format' : 'dd-MM-YYYY HH:mm:ss' > }, > "profile_image_url": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'string' > }, > "analysis": { > 'type': 'object', > 'index': 'analyzed', > 'store': 'yes', > 'properties': { > "hashtags": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object' > }, > "relations": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object' > }, > "score": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object' > } > } > }, > "location" : { > 'type': 'object', > 'index': 'analyzed', > 'store': 'yes', > "properties" : { > "search_field": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'analyzer': 'city_analyzer', > 'null_value': '' > }, > "name": { > 'index': 'not_analyzed', > 'store': 'yes', > 'type': 'string', > 'null_value': '' > }, > "city": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object', > 'properties': { > 'name': { > 'boost': 3.0, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'copy_to': 'location.search_field' > }, > 'full_name': { > 'boost': 3.0, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'copy_to': ['location.search_field', 'location.name'] > }, > 'alternate_names': { > 'boost': 2.0, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'copy_to': 'location.search_field' > } > } > }, > "admin2": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object', > 'properties': { > 'name': { > 'boost': 1.5, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'copy_to': 'location.search_field' > }, > 'full_name': { > 'boost': 1.5, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > 'copy_to': ['location.search_field', 'location.name'] > } > } > }, > "admin1": { > 'index': 'analyzed', > 'store': 'yes', > 'type': 'object', > 'properties': { > 'name': { > 'boost': 1.2, > 'index': 'analyzed', > 'store': 'yes', > 'type': 'string', > > ... -- You received this message because you are subscribed to the Google Groups "elasticsearch" group. To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/7875dd94-8b2a-405e-aaf4-eeb3c21bd53b%40googlegroups.com. For more options, visit https://groups.google.com/d/optout.