| # 2010 January 07 |
| # |
| # The author disclaims copyright to this source code. In place of |
| # a legal notice, here is a blessing: |
| # |
| # May you do good and not evil. |
| # May you find forgiveness for yourself and forgive others. |
| # May you share freely, never taking more than you give. |
| # |
| #************************************************************************* |
| # |
| # The tests in this file test the FTS3 auxillary functions offsets(), |
| # snippet() and matchinfo() work. At time of writing, running this file |
| # provides full coverage of fts3_snippet.c. |
| # |
| |
| set testdir [file dirname $argv0] |
| source $testdir/tester.tcl |
| |
| # If SQLITE_ENABLE_FTS3 is not defined, omit this file. |
| ifcapable !fts3 { finish_test ; return } |
| source $testdir/fts3_common.tcl |
| |
| set sqlite_fts3_enable_parentheses 1 |
| set DO_MALLOC_TEST 0 |
| |
| # Transform the list $L to its "normal" form. So that it can be compared to |
| # another list with the same set of elements using [string compare]. |
| # |
| proc normalize {L} { |
| set ret [list] |
| foreach l $L {lappend ret $l} |
| return $ret |
| } |
| |
| proc do_offsets_test {name expr args} { |
| set result [list] |
| foreach a $args { |
| lappend result [normalize $a] |
| } |
| do_select_test $name { |
| SELECT offsets(ft) FROM ft WHERE ft MATCH $expr |
| } $result |
| } |
| |
| # Document text used by a few tests. Contains the English names of all |
| # integers between 1 and 300. |
| # |
| set numbers [normalize { |
| one two three four five six seven eight nine ten eleven twelve thirteen |
| fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone |
| twentytwo twentythree twentyfour twentyfive twentysix twentyseven |
| twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour |
| thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone |
| fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight |
| fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix |
| fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour |
| sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone |
| seventytwo seventythree seventyfour seventyfive seventysix seventyseven |
| seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour |
| eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone |
| ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven |
| ninetyeight ninetynine onehundred onehundredone onehundredtwo |
| onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven |
| onehundredeight onehundrednine onehundredten onehundredeleven |
| onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen |
| onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen |
| onehundredtwenty onehundredtwentyone onehundredtwentytwo |
| onehundredtwentythree onehundredtwentyfour onehundredtwentyfive |
| onehundredtwentysix onehundredtwentyseven onehundredtwentyeight |
| onehundredtwentynine onehundredthirty onehundredthirtyone |
| onehundredthirtytwo onehundredthirtythree onehundredthirtyfour |
| onehundredthirtyfive onehundredthirtysix onehundredthirtyseven |
| onehundredthirtyeight onehundredthirtynine onehundredforty |
| onehundredfortyone onehundredfortytwo onehundredfortythree |
| onehundredfortyfour onehundredfortyfive onehundredfortysix |
| onehundredfortyseven onehundredfortyeight onehundredfortynine |
| onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree |
| onehundredfiftyfour onehundredfiftyfive onehundredfiftysix |
| onehundredfiftyseven onehundredfiftyeight onehundredfiftynine |
| onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree |
| onehundredsixtyfour onehundredsixtyfive onehundredsixtysix |
| onehundredsixtyseven onehundredsixtyeight onehundredsixtynine |
| onehundredseventy onehundredseventyone onehundredseventytwo |
| onehundredseventythree onehundredseventyfour onehundredseventyfive |
| onehundredseventysix onehundredseventyseven onehundredseventyeight |
| onehundredseventynine onehundredeighty onehundredeightyone |
| onehundredeightytwo onehundredeightythree onehundredeightyfour |
| onehundredeightyfive onehundredeightysix onehundredeightyseven |
| onehundredeightyeight onehundredeightynine onehundredninety |
| onehundredninetyone onehundredninetytwo onehundredninetythree |
| onehundredninetyfour onehundredninetyfive onehundredninetysix |
| onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred |
| twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive |
| twohundredsix twohundredseven twohundredeight twohundrednine twohundredten |
| twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen |
| twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen |
| twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo |
| twohundredtwentythree twohundredtwentyfour twohundredtwentyfive |
| twohundredtwentysix twohundredtwentyseven twohundredtwentyeight |
| twohundredtwentynine twohundredthirty twohundredthirtyone |
| twohundredthirtytwo twohundredthirtythree twohundredthirtyfour |
| twohundredthirtyfive twohundredthirtysix twohundredthirtyseven |
| twohundredthirtyeight twohundredthirtynine twohundredforty |
| twohundredfortyone twohundredfortytwo twohundredfortythree |
| twohundredfortyfour twohundredfortyfive twohundredfortysix |
| twohundredfortyseven twohundredfortyeight twohundredfortynine |
| twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree |
| twohundredfiftyfour twohundredfiftyfive twohundredfiftysix |
| twohundredfiftyseven twohundredfiftyeight twohundredfiftynine |
| twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree |
| twohundredsixtyfour twohundredsixtyfive twohundredsixtysix |
| twohundredsixtyseven twohundredsixtyeight twohundredsixtynine |
| twohundredseventy twohundredseventyone twohundredseventytwo |
| twohundredseventythree twohundredseventyfour twohundredseventyfive |
| twohundredseventysix twohundredseventyseven twohundredseventyeight |
| twohundredseventynine twohundredeighty twohundredeightyone |
| twohundredeightytwo twohundredeightythree twohundredeightyfour |
| twohundredeightyfive twohundredeightysix twohundredeightyseven |
| twohundredeightyeight twohundredeightynine twohundredninety |
| twohundredninetyone twohundredninetytwo twohundredninetythree |
| twohundredninetyfour twohundredninetyfive twohundredninetysix |
| twohundredninetyseven twohundredninetyeight twohundredninetynine |
| threehundred |
| }] |
| |
| foreach {DO_MALLOC_TEST enc} { |
| 0 utf8 |
| 1 utf8 |
| 1 utf16 |
| } { |
| |
| db close |
| file delete -force test.db |
| sqlite3 db test.db |
| sqlite3_db_config_lookaside db 0 0 0 |
| db eval "PRAGMA encoding = \"$enc\"" |
| |
| # Set variable $T to the test name prefix for this iteration of the loop. |
| # |
| set T "fts3snippet-$enc" |
| |
| ########################################################################## |
| # Test the offset function. |
| # |
| do_test $T.1.1 { |
| execsql { |
| CREATE VIRTUAL TABLE ft USING fts3; |
| INSERT INTO ft VALUES('xxx xxx xxx xxx'); |
| } |
| } {} |
| do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3} |
| do_offsets_test $T.1.3 {"xxx xxx"} { |
| 0 0 0 3 0 0 4 3 0 1 4 3 0 0 8 3 |
| 0 1 8 3 0 1 12 3 |
| } |
| do_offsets_test $T.1.4 {"xxx xxx" xxx} { |
| 0 0 0 3 0 2 0 3 0 0 4 3 0 1 4 3 |
| 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 |
| 0 1 12 3 0 2 12 3 |
| } |
| do_offsets_test $T.1.5 {xxx "xxx xxx"} { |
| 0 0 0 3 0 1 0 3 0 0 4 3 0 1 4 3 |
| 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 |
| 0 0 12 3 0 2 12 3 |
| } |
| |
| do_test $T.2.1 { |
| set v1 [lrange $numbers 0 99] |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3(a, b); |
| INSERT INTO ft VALUES($v1, $numbers); |
| INSERT INTO ft VALUES($v1, NULL); |
| } |
| } {} |
| |
| set off [string first "twohundred " $numbers] |
| do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10] |
| |
| set off [string first "onehundred " $numbers] |
| do_offsets_test $T.2.2 {onehundred} \ |
| [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10] |
| |
| # Test a corruption case: |
| execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers } |
| do_error_test $T.2.3 { |
| SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred' |
| } {database disk image is malformed} |
| |
| ########################################################################## |
| # Test the snippet function. |
| # |
| proc do_snippet_test {name expr iCol nTok args} { |
| set res [list] |
| foreach a $args { lappend res [string trim $a] } |
| do_select_test $name { |
| SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr |
| } $res |
| } |
| do_test $T.3.1 { |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3; |
| INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); |
| } |
| } {} |
| do_snippet_test $T.3.2 one 0 5 "{one} two three four five..." |
| do_snippet_test $T.3.3 two 0 5 "one {two} three four five..." |
| do_snippet_test $T.3.4 three 0 5 "one two {three} four five..." |
| do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..." |
| do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..." |
| do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..." |
| do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..." |
| do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten" |
| do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten" |
| do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}" |
| |
| do_test $T.4.1 { |
| execsql { |
| INSERT INTO ft VALUES( |
| 'one two three four five ' |
| || 'six seven eight nine ten ' |
| || 'eleven twelve thirteen fourteen fifteen ' |
| || 'sixteen seventeen eighteen nineteen twenty ' |
| || 'one two three four five ' |
| || 'six seven eight nine ten ' |
| || 'eleven twelve thirteen fourteen fifteen ' |
| || 'sixteen seventeen eighteen nineteen twenty' |
| ); |
| } |
| } {} |
| |
| do_snippet_test $T.4.2 {one nine} 0 5 { |
| {one} two three...eight {nine} ten |
| } { |
| {one} two three...eight {nine} ten... |
| } |
| |
| do_snippet_test $T.4.3 {one nine} 0 -5 { |
| {one} two three four five...six seven eight {nine} ten |
| } { |
| {one} two three four five...seven eight {nine} ten eleven... |
| } |
| do_snippet_test $T.4.3 {one nineteen} 0 -5 { |
| ...eighteen {nineteen} twenty {one} two... |
| } |
| do_snippet_test $T.4.4 {two nineteen} 0 -5 { |
| ...eighteen {nineteen} twenty one {two}... |
| } |
| do_snippet_test $T.4.5 {three nineteen} 0 -5 { |
| ...{nineteen} twenty one two {three}... |
| } |
| |
| do_snippet_test $T.4.6 {four nineteen} 0 -5 { |
| ...two three {four} five six...seventeen eighteen {nineteen} twenty one... |
| } |
| do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 { |
| ...seventeen eighteen {nineteen} twenty one...two three {four} five six... |
| } |
| |
| do_snippet_test $T.4.8 {four nineteen} 0 5 { |
| ...three {four} five...eighteen {nineteen} twenty... |
| } |
| do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 { |
| ...eighteen {nineteen} twenty...three {four} five... |
| } |
| do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 { |
| ...seventeen eighteen {nineteen} twenty one...two three {four} five six... |
| } |
| do_snippet_test $T.4.11 {four NOT (nineteen twentyone)} 0 5 { |
| ...two three {four} five six... |
| } { |
| ...two three {four} five six... |
| } |
| do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 { |
| ...two three {four} five six... |
| } { |
| ...two three {four} five six... |
| } |
| |
| do_test $T.5.1 { |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3(a, b, c); |
| INSERT INTO ft VALUES( |
| 'one two three four five', |
| 'four five six seven eight', |
| 'seven eight nine ten eleven' |
| ); |
| } |
| } {} |
| |
| do_snippet_test $T.5.2 {five} -1 3 {...three four {five}} |
| do_snippet_test $T.5.3 {five} 0 3 {...three four {five}} |
| do_snippet_test $T.5.4 {five} 1 3 {four {five} six...} |
| do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...} |
| |
| do_test $T.5.6 { |
| execsql { UPDATE ft SET b = NULL } |
| } {} |
| |
| do_snippet_test $T.5.7 {five} -1 3 {...three four {five}} |
| do_snippet_test $T.5.8 {five} 0 3 {...three four {five}} |
| do_snippet_test $T.5.9 {five} 1 3 {} |
| do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...} |
| |
| do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 { |
| {one} two three...{seven} {eight} {nine}... |
| } |
| |
| do_test $T.6.1 { |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3(x); |
| INSERT INTO ft VALUES($numbers); |
| } |
| } {} |
| do_snippet_test $T.6.2 { |
| one fifty onehundred onehundredfifty twohundredfifty threehundred |
| } -1 4 { |
| {one}...{fifty}...{onehundred}...{onehundredfifty}... |
| } |
| do_snippet_test $T.6.3 { |
| one fifty onehundred onehundredfifty twohundredfifty threehundred |
| } -1 -4 { |
| {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone... |
| } |
| |
| do_test $T.7.1 { |
| execsql { |
| BEGIN; |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3(x); |
| } |
| set testresults [list] |
| for {set i 1} {$i < 150} {incr i} { |
| set commas [string repeat , $i] |
| execsql {INSERT INTO ft VALUES('one' || $commas || 'two')} |
| lappend testresults "{one}$commas{two}" |
| } |
| execsql COMMIT |
| } {} |
| eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults |
| |
| ########################################################################## |
| # Test the matchinfo function. |
| # |
| proc mit {blob} { |
| set scan(littleEndian) i* |
| set scan(bigEndian) I* |
| binary scan $blob $scan($::tcl_platform(byteOrder)) r |
| return $r |
| } |
| db func mit mit |
| proc do_matchinfo_test {name expr args} { |
| set res [list] |
| foreach a $args { lappend res [normalize $a] } |
| do_select_test $name { |
| SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr |
| } $res |
| } |
| do_test $T.8.1 { |
| set ten {one two three four five six seven eight nine ten} |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3; |
| INSERT INTO ft VALUES($ten); |
| INSERT INTO ft VALUES($ten || ' ' || $ten); |
| } |
| } {} |
| |
| do_matchinfo_test $T.8.2 "one" {1 1 1 3 2} {1 1 2 3 2} |
| do_matchinfo_test $T.8.3 "one NEAR/3 ten" {2 1 1 1 1 1 1 1} |
| do_matchinfo_test $T.8.4 "five NEAR/4 ten" \ |
| {2 1 1 3 2 1 3 2} {2 1 2 3 2 2 3 2} |
| do_matchinfo_test $T.8.5 "six NEAR/3 ten NEAR/3 two" \ |
| {3 1 1 1 1 1 1 1 1 1 1} |
| do_matchinfo_test $T.8.6 "five NEAR/4 ten NEAR/3 two" \ |
| {3 1 2 2 1 1 1 1 1 1 1} |
| |
| do_test $T.9.1 { |
| execsql { |
| DROP TABLE IF EXISTS ft; |
| CREATE VIRTUAL TABLE ft USING fts3(x, y); |
| } |
| foreach n {1 2 3} { |
| set v1 [lrange $numbers 0 [expr $n*100]] |
| set v2 [string trim [string repeat "$numbers " $n]] |
| set docid [expr $n * 1000000] |
| execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) } |
| } |
| } {} |
| do_matchinfo_test $T.9.2 {two*} \ |
| { 1 2 1 105 3 101 606 3} \ |
| { 1 2 3 105 3 202 606 3} \ |
| { 1 2 101 105 3 303 606 3} |
| |
| do_matchinfo_test $T.9.4 {"one* two*"} \ |
| { 1 2 1 5 3 2 12 3} \ |
| { 1 2 2 5 3 4 12 3} \ |
| { 1 2 2 5 3 6 12 3} |
| |
| do_matchinfo_test $T.9.5 {twohundredfifty} \ |
| { 1 2 0 1 1 1 6 3} \ |
| { 1 2 0 1 1 2 6 3} \ |
| { 1 2 1 1 1 3 6 3} |
| |
| do_matchinfo_test $T.9.6 {"threehundred one"} \ |
| { 1 2 0 0 0 1 3 2} \ |
| { 1 2 0 0 0 2 3 2} |
| |
| do_matchinfo_test $T.9.7 {one OR fivehundred} \ |
| { 2 2 1 3 3 1 6 3 0 0 0 0 0 0 } \ |
| { 2 2 1 3 3 2 6 3 0 0 0 0 0 0 } \ |
| { 2 2 1 3 3 3 6 3 0 0 0 0 0 0 } |
| |
| do_matchinfo_test $T.9.8 {two OR "threehundred one"} \ |
| { 2 2 1 3 3 1 6 3 0 0 0 0 3 2 } \ |
| { 2 2 1 3 3 2 6 3 0 0 0 1 3 2 } \ |
| { 2 2 1 3 3 3 6 3 0 0 0 2 3 2 } |
| |
| do_select_test $T.9.9 { |
| SELECT mit(matchinfo(ft)), mit(matchinfo(ft)) |
| FROM ft WHERE ft MATCH 'two OR "threehundred one"' |
| } [normalize { |
| {2 2 1 3 3 1 6 3 0 0 0 0 3 2} |
| {2 2 1 3 3 1 6 3 0 0 0 0 3 2} |
| {2 2 1 3 3 2 6 3 0 0 0 1 3 2} |
| {2 2 1 3 3 2 6 3 0 0 0 1 3 2} |
| {2 2 1 3 3 3 6 3 0 0 0 2 3 2} |
| {2 2 1 3 3 3 6 3 0 0 0 2 3 2} |
| }] |
| |
| # EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the |
| # "query by rowid" or "linear scan" strategies, then the snippet and |
| # offsets both return an empty string, and the matchinfo function |
| # returns a blob value zero bytes in size. |
| # |
| set r 1000000 ;# A rowid that exists in table ft |
| do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r |
| do_select_test $T.10.1 { |
| SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft; |
| } {0 text 0 text 0 text} |
| do_select_test $T.10.2 { |
| SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft WHERE rowid = $r |
| } {0 text} |
| do_select_test $T.10.3 { |
| SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft; |
| } {0 text 0 text 0 text} |
| do_select_test $T.10.4 { |
| SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft WHERE rowid = $r; |
| } {0 text} |
| do_select_test $T.10.5 { |
| SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; |
| } {0 blob 0 blob 0 blob} |
| do_select_test $T.10.6 { |
| SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r |
| } {0 blob} |
| } |
| |
| set sqlite_fts3_enable_parentheses 0 |
| finish_test |
