diff --git a/data/converted/star_filled_svg.cpp b/data/converted/star_filled_svg.cpp index 3cec60910a..2557e6ca2c 100644 --- a/data/converted/star_filled_svg.cpp +++ b/data/converted/star_filled_svg.cpp @@ -2,123 +2,252 @@ #include "../Resources.h" -const size_t star_filled_svg_size = 1164; -const unsigned char star_filled_svg_data[1164] = { +const size_t star_filled_svg_size = 2444; +const unsigned char star_filled_svg_data[2444] = { 0x3c,0x3f,0x78,0x6d,0x6c,0x20,0x76,0x65,0x72,0x73, 0x69,0x6f,0x6e,0x3d,0x22,0x31,0x2e,0x30,0x22,0x20, 0x65,0x6e,0x63,0x6f,0x64,0x69,0x6e,0x67,0x3d,0x22, - 0x75,0x74,0x66,0x2d,0x38,0x22,0x3f,0x3e,0x0d,0x0a, - 0x3c,0x21,0x2d,0x2d,0x20,0x47,0x65,0x6e,0x65,0x72, - 0x61,0x74,0x6f,0x72,0x3a,0x20,0x41,0x64,0x6f,0x62, - 0x65,0x20,0x49,0x6c,0x6c,0x75,0x73,0x74,0x72,0x61, - 0x74,0x6f,0x72,0x20,0x31,0x36,0x2e,0x30,0x2e,0x33, - 0x2c,0x20,0x53,0x56,0x47,0x20,0x45,0x78,0x70,0x6f, - 0x72,0x74,0x20,0x50,0x6c,0x75,0x67,0x2d,0x49,0x6e, - 0x20,0x2e,0x20,0x53,0x56,0x47,0x20,0x56,0x65,0x72, - 0x73,0x69,0x6f,0x6e,0x3a,0x20,0x36,0x2e,0x30,0x30, - 0x20,0x42,0x75,0x69,0x6c,0x64,0x20,0x30,0x29,0x20, - 0x20,0x2d,0x2d,0x3e,0x0d,0x0a,0x3c,0x21,0x44,0x4f, - 0x43,0x54,0x59,0x50,0x45,0x20,0x73,0x76,0x67,0x20, - 0x50,0x55,0x42,0x4c,0x49,0x43,0x20,0x22,0x2d,0x2f, - 0x2f,0x57,0x33,0x43,0x2f,0x2f,0x44,0x54,0x44,0x20, - 0x53,0x56,0x47,0x20,0x31,0x2e,0x31,0x2f,0x2f,0x45, - 0x4e,0x22,0x20,0x22,0x68,0x74,0x74,0x70,0x3a,0x2f, - 0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72, - 0x67,0x2f,0x47,0x72,0x61,0x70,0x68,0x69,0x63,0x73, - 0x2f,0x53,0x56,0x47,0x2f,0x31,0x2e,0x31,0x2f,0x44, - 0x54,0x44,0x2f,0x73,0x76,0x67,0x31,0x31,0x2e,0x64, - 0x74,0x64,0x22,0x3e,0x0d,0x0a,0x3c,0x73,0x76,0x67, - 0x20,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3d,0x22, - 0x31,0x2e,0x31,0x22,0x20,0x69,0x64,0x3d,0x22,0x45, - 0x62,0x65,0x6e,0x65,0x5f,0x31,0x22,0x20,0x78,0x6d, - 0x6c,0x6e,0x73,0x3d,0x22,0x68,0x74,0x74,0x70,0x3a, - 0x2f,0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f, - 0x72,0x67,0x2f,0x32,0x30,0x30,0x30,0x2f,0x73,0x76, - 0x67,0x22,0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x78, - 0x6c,0x69,0x6e,0x6b,0x3d,0x22,0x68,0x74,0x74,0x70, - 0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e, - 0x6f,0x72,0x67,0x2f,0x31,0x39,0x39,0x39,0x2f,0x78, - 0x6c,0x69,0x6e,0x6b,0x22,0x20,0x78,0x3d,0x22,0x30, - 0x70,0x78,0x22,0x20,0x79,0x3d,0x22,0x30,0x70,0x78, - 0x22,0x0d,0x0a,0x09,0x20,0x77,0x69,0x64,0x74,0x68, - 0x3d,0x22,0x32,0x31,0x2e,0x37,0x37,0x35,0x70,0x78, - 0x22,0x20,0x68,0x65,0x69,0x67,0x68,0x74,0x3d,0x22, - 0x32,0x30,0x2e,0x37,0x36,0x32,0x70,0x78,0x22,0x20, - 0x76,0x69,0x65,0x77,0x42,0x6f,0x78,0x3d,0x22,0x30, - 0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37,0x35,0x20, - 0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x20,0x65,0x6e, - 0x61,0x62,0x6c,0x65,0x2d,0x62,0x61,0x63,0x6b,0x67, - 0x72,0x6f,0x75,0x6e,0x64,0x3d,0x22,0x6e,0x65,0x77, - 0x20,0x30,0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37, - 0x35,0x20,0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x20, - 0x78,0x6d,0x6c,0x3a,0x73,0x70,0x61,0x63,0x65,0x3d, - 0x22,0x70,0x72,0x65,0x73,0x65,0x72,0x76,0x65,0x22, - 0x3e,0x0d,0x0a,0x3c,0x70,0x61,0x74,0x68,0x20,0x66, - 0x69,0x6c,0x6c,0x3d,0x22,0x23,0x37,0x37,0x37,0x37, - 0x37,0x37,0x22,0x20,0x64,0x3d,0x22,0x4d,0x38,0x2e, - 0x37,0x31,0x2c,0x32,0x2e,0x30,0x37,0x36,0x63,0x2d, - 0x30,0x2e,0x31,0x38,0x32,0x2c,0x30,0x2d,0x30,0x2e, - 0x33,0x36,0x34,0x2c,0x30,0x2e,0x31,0x34,0x2d,0x30, + 0x55,0x54,0x46,0x2d,0x38,0x22,0x20,0x73,0x74,0x61, + 0x6e,0x64,0x61,0x6c,0x6f,0x6e,0x65,0x3d,0x22,0x6e, + 0x6f,0x22,0x3f,0x3e,0x0a,0x3c,0x21,0x2d,0x2d,0x20, + 0x47,0x65,0x6e,0x65,0x72,0x61,0x74,0x6f,0x72,0x3a, + 0x20,0x41,0x64,0x6f,0x62,0x65,0x20,0x49,0x6c,0x6c, + 0x75,0x73,0x74,0x72,0x61,0x74,0x6f,0x72,0x20,0x31, + 0x36,0x2e,0x30,0x2e,0x33,0x2c,0x20,0x53,0x56,0x47, + 0x20,0x45,0x78,0x70,0x6f,0x72,0x74,0x20,0x50,0x6c, + 0x75,0x67,0x2d,0x49,0x6e,0x20,0x2e,0x20,0x53,0x56, + 0x47,0x20,0x56,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3a, + 0x20,0x36,0x2e,0x30,0x30,0x20,0x42,0x75,0x69,0x6c, + 0x64,0x20,0x30,0x29,0x20,0x20,0x2d,0x2d,0x3e,0x0a, + 0x0a,0x3c,0x73,0x76,0x67,0x0a,0x20,0x20,0x20,0x78, + 0x6d,0x6c,0x6e,0x73,0x3a,0x64,0x63,0x3d,0x22,0x68, + 0x74,0x74,0x70,0x3a,0x2f,0x2f,0x70,0x75,0x72,0x6c, + 0x2e,0x6f,0x72,0x67,0x2f,0x64,0x63,0x2f,0x65,0x6c, + 0x65,0x6d,0x65,0x6e,0x74,0x73,0x2f,0x31,0x2e,0x31, + 0x2f,0x22,0x0a,0x20,0x20,0x20,0x78,0x6d,0x6c,0x6e, + 0x73,0x3a,0x63,0x63,0x3d,0x22,0x68,0x74,0x74,0x70, + 0x3a,0x2f,0x2f,0x63,0x72,0x65,0x61,0x74,0x69,0x76, + 0x65,0x63,0x6f,0x6d,0x6d,0x6f,0x6e,0x73,0x2e,0x6f, + 0x72,0x67,0x2f,0x6e,0x73,0x23,0x22,0x0a,0x20,0x20, + 0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x72,0x64,0x66, + 0x3d,0x22,0x68,0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77, + 0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f, + 0x31,0x39,0x39,0x39,0x2f,0x30,0x32,0x2f,0x32,0x32, + 0x2d,0x72,0x64,0x66,0x2d,0x73,0x79,0x6e,0x74,0x61, + 0x78,0x2d,0x6e,0x73,0x23,0x22,0x0a,0x20,0x20,0x20, + 0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x73,0x76,0x67,0x3d, + 0x22,0x68,0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77,0x77, + 0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f,0x32, + 0x30,0x30,0x30,0x2f,0x73,0x76,0x67,0x22,0x0a,0x20, + 0x20,0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3d,0x22,0x68, + 0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e, + 0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f,0x32,0x30,0x30, + 0x30,0x2f,0x73,0x76,0x67,0x22,0x0a,0x20,0x20,0x20, + 0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x73,0x6f,0x64,0x69, + 0x70,0x6f,0x64,0x69,0x3d,0x22,0x68,0x74,0x74,0x70, + 0x3a,0x2f,0x2f,0x73,0x6f,0x64,0x69,0x70,0x6f,0x64, + 0x69,0x2e,0x73,0x6f,0x75,0x72,0x63,0x65,0x66,0x6f, + 0x72,0x67,0x65,0x2e,0x6e,0x65,0x74,0x2f,0x44,0x54, + 0x44,0x2f,0x73,0x6f,0x64,0x69,0x70,0x6f,0x64,0x69, + 0x2d,0x30,0x2e,0x64,0x74,0x64,0x22,0x0a,0x20,0x20, + 0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x69,0x6e,0x6b, + 0x73,0x63,0x61,0x70,0x65,0x3d,0x22,0x68,0x74,0x74, + 0x70,0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e,0x69,0x6e, + 0x6b,0x73,0x63,0x61,0x70,0x65,0x2e,0x6f,0x72,0x67, + 0x2f,0x6e,0x61,0x6d,0x65,0x73,0x70,0x61,0x63,0x65, + 0x73,0x2f,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x22,0x0a,0x20,0x20,0x20,0x76,0x65,0x72,0x73,0x69, + 0x6f,0x6e,0x3d,0x22,0x31,0x2e,0x31,0x22,0x0a,0x20, + 0x20,0x20,0x69,0x64,0x3d,0x22,0x45,0x62,0x65,0x6e, + 0x65,0x5f,0x31,0x22,0x0a,0x20,0x20,0x20,0x78,0x3d, + 0x22,0x30,0x70,0x78,0x22,0x0a,0x20,0x20,0x20,0x79, + 0x3d,0x22,0x30,0x70,0x78,0x22,0x0a,0x20,0x20,0x20, + 0x77,0x69,0x64,0x74,0x68,0x3d,0x22,0x32,0x31,0x2e, + 0x37,0x37,0x35,0x70,0x78,0x22,0x0a,0x20,0x20,0x20, + 0x68,0x65,0x69,0x67,0x68,0x74,0x3d,0x22,0x32,0x30, + 0x2e,0x37,0x36,0x32,0x70,0x78,0x22,0x0a,0x20,0x20, + 0x20,0x76,0x69,0x65,0x77,0x42,0x6f,0x78,0x3d,0x22, + 0x30,0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37,0x35, + 0x20,0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x0a,0x20, + 0x20,0x20,0x65,0x6e,0x61,0x62,0x6c,0x65,0x2d,0x62, + 0x61,0x63,0x6b,0x67,0x72,0x6f,0x75,0x6e,0x64,0x3d, + 0x22,0x6e,0x65,0x77,0x20,0x30,0x20,0x30,0x20,0x32, + 0x31,0x2e,0x37,0x37,0x35,0x20,0x32,0x30,0x2e,0x37, + 0x36,0x32,0x22,0x0a,0x20,0x20,0x20,0x78,0x6d,0x6c, + 0x3a,0x73,0x70,0x61,0x63,0x65,0x3d,0x22,0x70,0x72, + 0x65,0x73,0x65,0x72,0x76,0x65,0x22,0x0a,0x20,0x20, + 0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a, + 0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3d,0x22,0x30, + 0x2e,0x39,0x31,0x20,0x72,0x31,0x33,0x37,0x32,0x35, + 0x22,0x0a,0x20,0x20,0x20,0x73,0x6f,0x64,0x69,0x70, + 0x6f,0x64,0x69,0x3a,0x64,0x6f,0x63,0x6e,0x61,0x6d, + 0x65,0x3d,0x22,0x73,0x74,0x61,0x72,0x5f,0x66,0x69, + 0x6c,0x6c,0x65,0x64,0x2e,0x73,0x76,0x67,0x22,0x3e, + 0x3c,0x6d,0x65,0x74,0x61,0x64,0x61,0x74,0x61,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x69,0x64,0x3d,0x22,0x6d, + 0x65,0x74,0x61,0x64,0x61,0x74,0x61,0x33,0x33,0x39, + 0x39,0x22,0x3e,0x3c,0x72,0x64,0x66,0x3a,0x52,0x44, + 0x46,0x3e,0x3c,0x63,0x63,0x3a,0x57,0x6f,0x72,0x6b, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x72,0x64,0x66,0x3a,0x61,0x62,0x6f,0x75,0x74,0x3d, + 0x22,0x22,0x3e,0x3c,0x64,0x63,0x3a,0x66,0x6f,0x72, + 0x6d,0x61,0x74,0x3e,0x69,0x6d,0x61,0x67,0x65,0x2f, + 0x73,0x76,0x67,0x2b,0x78,0x6d,0x6c,0x3c,0x2f,0x64, + 0x63,0x3a,0x66,0x6f,0x72,0x6d,0x61,0x74,0x3e,0x3c, + 0x64,0x63,0x3a,0x74,0x79,0x70,0x65,0x0a,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x72, + 0x64,0x66,0x3a,0x72,0x65,0x73,0x6f,0x75,0x72,0x63, + 0x65,0x3d,0x22,0x68,0x74,0x74,0x70,0x3a,0x2f,0x2f, + 0x70,0x75,0x72,0x6c,0x2e,0x6f,0x72,0x67,0x2f,0x64, + 0x63,0x2f,0x64,0x63,0x6d,0x69,0x74,0x79,0x70,0x65, + 0x2f,0x53,0x74,0x69,0x6c,0x6c,0x49,0x6d,0x61,0x67, + 0x65,0x22,0x20,0x2f,0x3e,0x3c,0x64,0x63,0x3a,0x74, + 0x69,0x74,0x6c,0x65,0x3e,0x3c,0x2f,0x64,0x63,0x3a, + 0x74,0x69,0x74,0x6c,0x65,0x3e,0x3c,0x2f,0x63,0x63, + 0x3a,0x57,0x6f,0x72,0x6b,0x3e,0x3c,0x2f,0x72,0x64, + 0x66,0x3a,0x52,0x44,0x46,0x3e,0x3c,0x2f,0x6d,0x65, + 0x74,0x61,0x64,0x61,0x74,0x61,0x3e,0x3c,0x64,0x65, + 0x66,0x73,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64, + 0x3d,0x22,0x64,0x65,0x66,0x73,0x33,0x33,0x39,0x37, + 0x22,0x20,0x2f,0x3e,0x3c,0x73,0x6f,0x64,0x69,0x70, + 0x6f,0x64,0x69,0x3a,0x6e,0x61,0x6d,0x65,0x64,0x76, + 0x69,0x65,0x77,0x0a,0x20,0x20,0x20,0x20,0x20,0x70, + 0x61,0x67,0x65,0x63,0x6f,0x6c,0x6f,0x72,0x3d,0x22, + 0x23,0x66,0x66,0x66,0x66,0x66,0x66,0x22,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x62,0x6f,0x72,0x64,0x65,0x72, + 0x63,0x6f,0x6c,0x6f,0x72,0x3d,0x22,0x23,0x36,0x36, + 0x36,0x36,0x36,0x36,0x22,0x0a,0x20,0x20,0x20,0x20, + 0x20,0x62,0x6f,0x72,0x64,0x65,0x72,0x6f,0x70,0x61, + 0x63,0x69,0x74,0x79,0x3d,0x22,0x31,0x22,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x6f,0x62,0x6a,0x65,0x63,0x74, + 0x74,0x6f,0x6c,0x65,0x72,0x61,0x6e,0x63,0x65,0x3d, + 0x22,0x31,0x30,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x67,0x72,0x69,0x64,0x74,0x6f,0x6c,0x65,0x72,0x61, + 0x6e,0x63,0x65,0x3d,0x22,0x31,0x30,0x22,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x67,0x75,0x69,0x64,0x65,0x74, + 0x6f,0x6c,0x65,0x72,0x61,0x6e,0x63,0x65,0x3d,0x22, + 0x31,0x30,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69, + 0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x70,0x61, + 0x67,0x65,0x6f,0x70,0x61,0x63,0x69,0x74,0x79,0x3d, + 0x22,0x30,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69, + 0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x70,0x61, + 0x67,0x65,0x73,0x68,0x61,0x64,0x6f,0x77,0x3d,0x22, + 0x32,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e, + 0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x77,0x69,0x6e, + 0x64,0x6f,0x77,0x2d,0x77,0x69,0x64,0x74,0x68,0x3d, + 0x22,0x31,0x35,0x33,0x31,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x3a,0x77,0x69,0x6e,0x64,0x6f,0x77,0x2d,0x68,0x65, + 0x69,0x67,0x68,0x74,0x3d,0x22,0x38,0x37,0x31,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64,0x3d,0x22, + 0x6e,0x61,0x6d,0x65,0x64,0x76,0x69,0x65,0x77,0x33, + 0x33,0x39,0x35,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x73,0x68,0x6f,0x77,0x67,0x72,0x69,0x64,0x3d,0x22, + 0x66,0x61,0x6c,0x73,0x65,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x3a,0x7a,0x6f,0x6f,0x6d,0x3d,0x22,0x33,0x33,0x2e, + 0x30,0x34,0x31,0x31,0x33,0x34,0x22,0x0a,0x20,0x20, + 0x20,0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70, + 0x65,0x3a,0x63,0x78,0x3d,0x22,0x37,0x2e,0x30,0x35, + 0x38,0x39,0x33,0x38,0x37,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x3a,0x63,0x79,0x3d,0x22,0x31,0x30,0x2e,0x33,0x38, + 0x31,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e, + 0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x77,0x69,0x6e, + 0x64,0x6f,0x77,0x2d,0x78,0x3d,0x22,0x36,0x39,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e,0x6b,0x73, + 0x63,0x61,0x70,0x65,0x3a,0x77,0x69,0x6e,0x64,0x6f, + 0x77,0x2d,0x79,0x3d,0x22,0x30,0x22,0x0a,0x20,0x20, + 0x20,0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70, + 0x65,0x3a,0x77,0x69,0x6e,0x64,0x6f,0x77,0x2d,0x6d, + 0x61,0x78,0x69,0x6d,0x69,0x7a,0x65,0x64,0x3d,0x22, + 0x31,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e, + 0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x63,0x75,0x72, + 0x72,0x65,0x6e,0x74,0x2d,0x6c,0x61,0x79,0x65,0x72, + 0x3d,0x22,0x45,0x62,0x65,0x6e,0x65,0x5f,0x31,0x22, + 0x20,0x2f,0x3e,0x3c,0x70,0x61,0x74,0x68,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x64,0x3d,0x22,0x6d,0x20,0x31, + 0x30,0x2e,0x39,0x35,0x39,0x39,0x39,0x39,0x2c,0x32, + 0x2e,0x30,0x37,0x36,0x20,0x63,0x20,0x2d,0x30,0x2e, + 0x31,0x38,0x32,0x2c,0x30,0x20,0x2d,0x30,0x2e,0x33, + 0x36,0x34,0x2c,0x30,0x2e,0x31,0x34,0x20,0x2d,0x30, 0x2e,0x35,0x30,0x31,0x2c,0x30,0x2e,0x34,0x31,0x39, - 0x4c,0x36,0x2e,0x33,0x35,0x2c,0x36,0x2e,0x32,0x36, - 0x31,0x63,0x2d,0x30,0x2e,0x32,0x37,0x36,0x2c,0x30, - 0x2e,0x35,0x36,0x2d,0x31,0x2e,0x30,0x30,0x37,0x2c, - 0x31,0x2e,0x30,0x39,0x31,0x2d,0x31,0x2e,0x36,0x32, - 0x34,0x2c,0x31,0x2e,0x31,0x38,0x4c,0x30,0x2e,0x35, - 0x37,0x32,0x2c,0x38,0x2e,0x30,0x34,0x35,0x0d,0x0a, - 0x09,0x43,0x2d,0x30,0x2e,0x30,0x34,0x36,0x2c,0x38, - 0x2e,0x31,0x33,0x35,0x2d,0x30,0x2e,0x31,0x38,0x35, - 0x2c,0x38,0x2e,0x35,0x36,0x34,0x2c,0x30,0x2e,0x32, - 0x36,0x32,0x2c,0x39,0x6c,0x33,0x2e,0x30,0x30,0x37, - 0x2c,0x32,0x2e,0x39,0x33,0x31,0x63,0x30,0x2e,0x34, - 0x34,0x37,0x2c,0x30,0x2e,0x34,0x33,0x36,0x2c,0x30, - 0x2e,0x37,0x32,0x36,0x2c,0x31,0x2e,0x32,0x39,0x35, - 0x2c,0x30,0x2e,0x36,0x32,0x2c,0x31,0x2e,0x39,0x30, - 0x39,0x6c,0x2d,0x30,0x2e,0x37,0x31,0x2c,0x34,0x2e, - 0x31,0x34,0x0d,0x0a,0x09,0x63,0x2d,0x30,0x2e,0x30, - 0x37,0x37,0x2c,0x30,0x2e,0x34,0x34,0x36,0x2c,0x30, - 0x2e,0x30,0x39,0x34,0x2c,0x30,0x2e,0x37,0x30,0x38, - 0x2c,0x30,0x2e,0x34,0x30,0x35,0x2c,0x30,0x2e,0x37, - 0x30,0x38,0x63,0x30,0x2e,0x31,0x31,0x38,0x2c,0x30, - 0x2c,0x30,0x2e,0x32,0x35,0x35,0x2d,0x30,0x2e,0x30, - 0x33,0x38,0x2c,0x30,0x2e,0x34,0x30,0x37,0x2d,0x30, - 0x2e,0x31,0x31,0x37,0x6c,0x33,0x2e,0x37,0x31,0x37, - 0x2d,0x31,0x2e,0x39,0x35,0x36,0x63,0x30,0x2e,0x32, - 0x37,0x36,0x2d,0x30,0x2e,0x31,0x34,0x35,0x2c,0x30, - 0x2e,0x36,0x34,0x2d,0x30,0x2e,0x32,0x31,0x36,0x2c, - 0x31,0x2e,0x30,0x30,0x33,0x2d,0x30,0x2e,0x32,0x31, - 0x36,0x0d,0x0a,0x09,0x63,0x30,0x2e,0x33,0x36,0x34, - 0x2c,0x30,0x2c,0x30,0x2e,0x37,0x32,0x39,0x2c,0x30, - 0x2e,0x30,0x37,0x32,0x2c,0x31,0x2e,0x30,0x30,0x34, - 0x2c,0x30,0x2e,0x32,0x31,0x36,0x6c,0x33,0x2e,0x37, - 0x31,0x37,0x2c,0x31,0x2e,0x39,0x35,0x36,0x63,0x30, - 0x2e,0x31,0x35,0x2c,0x30,0x2e,0x30,0x37,0x39,0x2c, - 0x30,0x2e,0x32,0x38,0x39,0x2c,0x30,0x2e,0x31,0x31, - 0x37,0x2c,0x30,0x2e,0x34,0x30,0x37,0x2c,0x30,0x2e, - 0x31,0x31,0x37,0x63,0x30,0x2e,0x33,0x31,0x31,0x2c, - 0x30,0x2c,0x30,0x2e,0x34,0x38,0x31,0x2d,0x30,0x2e, - 0x32,0x36,0x32,0x2c,0x30,0x2e,0x34,0x30,0x35,0x2d, - 0x30,0x2e,0x37,0x30,0x38,0x6c,0x2d,0x30,0x2e,0x37, - 0x31,0x31,0x2d,0x34,0x2e,0x31,0x34,0x0d,0x0a,0x09, - 0x63,0x2d,0x30,0x2e,0x31,0x30,0x35,0x2d,0x30,0x2e, - 0x36,0x31,0x34,0x2c,0x30,0x2e,0x31,0x37,0x34,0x2d, - 0x31,0x2e,0x34,0x37,0x34,0x2c,0x30,0x2e,0x36,0x31, - 0x39,0x2d,0x31,0x2e,0x39,0x30,0x39,0x4c,0x31,0x37, - 0x2e,0x31,0x36,0x2c,0x39,0x63,0x30,0x2e,0x34,0x34, - 0x36,0x2d,0x30,0x2e,0x34,0x33,0x36,0x2c,0x30,0x2e, - 0x33,0x30,0x37,0x2d,0x30,0x2e,0x38,0x36,0x35,0x2d, - 0x30,0x2e,0x33,0x31,0x31,0x2d,0x30,0x2e,0x39,0x35, - 0x35,0x6c,0x2d,0x34,0x2e,0x31,0x35,0x34,0x2d,0x30, - 0x2e,0x36,0x30,0x33,0x0d,0x0a,0x09,0x63,0x2d,0x30, - 0x2e,0x36,0x31,0x37,0x2d,0x30,0x2e,0x30,0x38,0x39, - 0x2d,0x31,0x2e,0x33,0x34,0x39,0x2d,0x30,0x2e,0x36, - 0x32,0x2d,0x31,0x2e,0x36,0x32,0x34,0x2d,0x31,0x2e, - 0x31,0x38,0x4c,0x39,0x2e,0x32,0x31,0x33,0x2c,0x32, - 0x2e,0x34,0x39,0x35,0x43,0x39,0x2e,0x30,0x37,0x34, - 0x2c,0x32,0x2e,0x32,0x31,0x36,0x2c,0x38,0x2e,0x38, - 0x39,0x33,0x2c,0x32,0x2e,0x30,0x37,0x36,0x2c,0x38, - 0x2e,0x37,0x31,0x2c,0x32,0x2e,0x30,0x37,0x36,0x4c, - 0x38,0x2e,0x37,0x31,0x2c,0x32,0x2e,0x30,0x37,0x36, - 0x7a,0x22,0x2f,0x3e,0x0d,0x0a,0x3c,0x2f,0x73,0x76, - 0x67,0x3e,0x0d,0x0a + 0x20,0x4c,0x20,0x38,0x2e,0x35,0x39,0x39,0x39,0x39, + 0x38,0x32,0x2c,0x36,0x2e,0x32,0x36,0x31,0x20,0x63, + 0x20,0x2d,0x30,0x2e,0x32,0x37,0x36,0x2c,0x30,0x2e, + 0x35,0x36,0x20,0x2d,0x31,0x2e,0x30,0x30,0x37,0x2c, + 0x31,0x2e,0x30,0x39,0x31,0x20,0x2d,0x31,0x2e,0x36, + 0x32,0x34,0x2c,0x31,0x2e,0x31,0x38,0x20,0x6c,0x20, + 0x2d,0x34,0x2e,0x31,0x35,0x34,0x2c,0x30,0x2e,0x36, + 0x30,0x34,0x20,0x63,0x20,0x2d,0x30,0x2e,0x36,0x31, + 0x38,0x2c,0x30,0x2e,0x30,0x39,0x20,0x2d,0x30,0x2e, + 0x37,0x35,0x37,0x2c,0x30,0x2e,0x35,0x31,0x39,0x20, + 0x2d,0x30,0x2e,0x33,0x31,0x2c,0x30,0x2e,0x39,0x35, + 0x35,0x20,0x6c,0x20,0x33,0x2e,0x30,0x30,0x37,0x2c, + 0x32,0x2e,0x39,0x33,0x31,0x20,0x63,0x20,0x30,0x2e, + 0x34,0x34,0x37,0x2c,0x30,0x2e,0x34,0x33,0x36,0x20, + 0x30,0x2e,0x37,0x32,0x36,0x2c,0x31,0x2e,0x32,0x39, + 0x35,0x20,0x30,0x2e,0x36,0x32,0x2c,0x31,0x2e,0x39, + 0x30,0x39,0x20,0x6c,0x20,0x2d,0x30,0x2e,0x37,0x31, + 0x2c,0x34,0x2e,0x31,0x34,0x20,0x63,0x20,0x2d,0x30, + 0x2e,0x30,0x37,0x37,0x2c,0x30,0x2e,0x34,0x34,0x36, + 0x20,0x30,0x2e,0x30,0x39,0x34,0x2c,0x30,0x2e,0x37, + 0x30,0x38,0x20,0x30,0x2e,0x34,0x30,0x35,0x2c,0x30, + 0x2e,0x37,0x30,0x38,0x20,0x30,0x2e,0x31,0x31,0x38, + 0x2c,0x30,0x20,0x30,0x2e,0x32,0x35,0x35,0x2c,0x2d, + 0x30,0x2e,0x30,0x33,0x38,0x20,0x30,0x2e,0x34,0x30, + 0x37,0x2c,0x2d,0x30,0x2e,0x31,0x31,0x37,0x20,0x6c, + 0x20,0x33,0x2e,0x37,0x31,0x37,0x2c,0x2d,0x31,0x2e, + 0x39,0x35,0x36,0x20,0x63,0x20,0x30,0x2e,0x32,0x37, + 0x36,0x30,0x30,0x30,0x38,0x2c,0x2d,0x30,0x2e,0x31, + 0x34,0x35,0x20,0x30,0x2e,0x36,0x34,0x30,0x30,0x30, + 0x30,0x38,0x2c,0x2d,0x30,0x2e,0x32,0x31,0x36,0x20, + 0x31,0x2e,0x30,0x30,0x33,0x30,0x30,0x30,0x38,0x2c, + 0x2d,0x30,0x2e,0x32,0x31,0x36,0x20,0x30,0x2e,0x33, + 0x36,0x34,0x2c,0x30,0x20,0x30,0x2e,0x37,0x32,0x39, + 0x2c,0x30,0x2e,0x30,0x37,0x32,0x20,0x31,0x2e,0x30, + 0x30,0x34,0x2c,0x30,0x2e,0x32,0x31,0x36,0x20,0x6c, + 0x20,0x33,0x2e,0x37,0x31,0x37,0x2c,0x31,0x2e,0x39, + 0x35,0x36,0x20,0x63,0x20,0x30,0x2e,0x31,0x35,0x2c, + 0x30,0x2e,0x30,0x37,0x39,0x20,0x30,0x2e,0x32,0x38, + 0x39,0x2c,0x30,0x2e,0x31,0x31,0x37,0x20,0x30,0x2e, + 0x34,0x30,0x37,0x2c,0x30,0x2e,0x31,0x31,0x37,0x20, + 0x30,0x2e,0x33,0x31,0x31,0x2c,0x30,0x20,0x30,0x2e, + 0x34,0x38,0x31,0x2c,0x2d,0x30,0x2e,0x32,0x36,0x32, + 0x20,0x30,0x2e,0x34,0x30,0x35,0x2c,0x2d,0x30,0x2e, + 0x37,0x30,0x38,0x20,0x6c,0x20,0x2d,0x30,0x2e,0x37, + 0x31,0x31,0x2c,0x2d,0x34,0x2e,0x31,0x34,0x20,0x63, + 0x20,0x2d,0x30,0x2e,0x31,0x30,0x35,0x2c,0x2d,0x30, + 0x2e,0x36,0x31,0x34,0x20,0x30,0x2e,0x31,0x37,0x34, + 0x2c,0x2d,0x31,0x2e,0x34,0x37,0x34,0x20,0x30,0x2e, + 0x36,0x31,0x39,0x2c,0x2d,0x31,0x2e,0x39,0x30,0x39, + 0x20,0x4c,0x20,0x31,0x39,0x2e,0x34,0x30,0x39,0x39, + 0x39,0x39,0x2c,0x39,0x20,0x63,0x20,0x30,0x2e,0x34, + 0x34,0x36,0x2c,0x2d,0x30,0x2e,0x34,0x33,0x36,0x20, + 0x30,0x2e,0x33,0x30,0x37,0x2c,0x2d,0x30,0x2e,0x38, + 0x36,0x35,0x20,0x2d,0x30,0x2e,0x33,0x31,0x31,0x2c, + 0x2d,0x30,0x2e,0x39,0x35,0x35,0x20,0x6c,0x20,0x2d, + 0x34,0x2e,0x31,0x35,0x34,0x2c,0x2d,0x30,0x2e,0x36, + 0x30,0x33,0x20,0x63,0x20,0x2d,0x30,0x2e,0x36,0x31, + 0x37,0x2c,0x2d,0x30,0x2e,0x30,0x38,0x39,0x20,0x2d, + 0x31,0x2e,0x33,0x34,0x39,0x2c,0x2d,0x30,0x2e,0x36, + 0x32,0x20,0x2d,0x31,0x2e,0x36,0x32,0x34,0x2c,0x2d, + 0x31,0x2e,0x31,0x38,0x20,0x6c,0x20,0x2d,0x31,0x2e, + 0x38,0x35,0x38,0x2c,0x2d,0x33,0x2e,0x37,0x36,0x37, + 0x20,0x63,0x20,0x2d,0x30,0x2e,0x31,0x33,0x39,0x2c, + 0x2d,0x30,0x2e,0x32,0x37,0x39,0x20,0x2d,0x30,0x2e, + 0x33,0x32,0x2c,0x2d,0x30,0x2e,0x34,0x31,0x39,0x20, + 0x2d,0x30,0x2e,0x35,0x30,0x33,0x2c,0x2d,0x30,0x2e, + 0x34,0x31,0x39,0x20,0x6c,0x20,0x30,0x2c,0x30,0x20, + 0x7a,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64, + 0x3d,0x22,0x70,0x61,0x74,0x68,0x33,0x33,0x39,0x33, + 0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e,0x6b, + 0x73,0x63,0x61,0x70,0x65,0x3a,0x63,0x6f,0x6e,0x6e, + 0x65,0x63,0x74,0x6f,0x72,0x2d,0x63,0x75,0x72,0x76, + 0x61,0x74,0x75,0x72,0x65,0x3d,0x22,0x30,0x22,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x73,0x74,0x79,0x6c,0x65, + 0x3d,0x22,0x66,0x69,0x6c,0x6c,0x3a,0x23,0x37,0x37, + 0x37,0x37,0x37,0x37,0x22,0x20,0x2f,0x3e,0x3c,0x2f, + 0x73,0x76,0x67,0x3e }; + diff --git a/data/converted/star_unfilled_svg.cpp b/data/converted/star_unfilled_svg.cpp index 5d6b43bda3..d897cedfbf 100644 --- a/data/converted/star_unfilled_svg.cpp +++ b/data/converted/star_unfilled_svg.cpp @@ -2,195 +2,332 @@ #include "../Resources.h" -const size_t star_unfilled_svg_size = 1889; -const unsigned char star_unfilled_svg_data[1889] = { +const size_t star_unfilled_svg_size = 3243; +const unsigned char star_unfilled_svg_data[3243] = { 0x3c,0x3f,0x78,0x6d,0x6c,0x20,0x76,0x65,0x72,0x73, 0x69,0x6f,0x6e,0x3d,0x22,0x31,0x2e,0x30,0x22,0x20, 0x65,0x6e,0x63,0x6f,0x64,0x69,0x6e,0x67,0x3d,0x22, - 0x75,0x74,0x66,0x2d,0x38,0x22,0x3f,0x3e,0x0d,0x0a, - 0x3c,0x21,0x2d,0x2d,0x20,0x47,0x65,0x6e,0x65,0x72, - 0x61,0x74,0x6f,0x72,0x3a,0x20,0x41,0x64,0x6f,0x62, - 0x65,0x20,0x49,0x6c,0x6c,0x75,0x73,0x74,0x72,0x61, - 0x74,0x6f,0x72,0x20,0x31,0x36,0x2e,0x30,0x2e,0x33, - 0x2c,0x20,0x53,0x56,0x47,0x20,0x45,0x78,0x70,0x6f, - 0x72,0x74,0x20,0x50,0x6c,0x75,0x67,0x2d,0x49,0x6e, - 0x20,0x2e,0x20,0x53,0x56,0x47,0x20,0x56,0x65,0x72, - 0x73,0x69,0x6f,0x6e,0x3a,0x20,0x36,0x2e,0x30,0x30, - 0x20,0x42,0x75,0x69,0x6c,0x64,0x20,0x30,0x29,0x20, - 0x20,0x2d,0x2d,0x3e,0x0d,0x0a,0x3c,0x21,0x44,0x4f, - 0x43,0x54,0x59,0x50,0x45,0x20,0x73,0x76,0x67,0x20, - 0x50,0x55,0x42,0x4c,0x49,0x43,0x20,0x22,0x2d,0x2f, - 0x2f,0x57,0x33,0x43,0x2f,0x2f,0x44,0x54,0x44,0x20, - 0x53,0x56,0x47,0x20,0x31,0x2e,0x31,0x2f,0x2f,0x45, - 0x4e,0x22,0x20,0x22,0x68,0x74,0x74,0x70,0x3a,0x2f, - 0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72, - 0x67,0x2f,0x47,0x72,0x61,0x70,0x68,0x69,0x63,0x73, - 0x2f,0x53,0x56,0x47,0x2f,0x31,0x2e,0x31,0x2f,0x44, - 0x54,0x44,0x2f,0x73,0x76,0x67,0x31,0x31,0x2e,0x64, - 0x74,0x64,0x22,0x3e,0x0d,0x0a,0x3c,0x73,0x76,0x67, - 0x20,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3d,0x22, - 0x31,0x2e,0x31,0x22,0x20,0x69,0x64,0x3d,0x22,0x45, - 0x62,0x65,0x6e,0x65,0x5f,0x31,0x22,0x20,0x78,0x6d, - 0x6c,0x6e,0x73,0x3d,0x22,0x68,0x74,0x74,0x70,0x3a, - 0x2f,0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f, - 0x72,0x67,0x2f,0x32,0x30,0x30,0x30,0x2f,0x73,0x76, - 0x67,0x22,0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x78, - 0x6c,0x69,0x6e,0x6b,0x3d,0x22,0x68,0x74,0x74,0x70, - 0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e,0x77,0x33,0x2e, - 0x6f,0x72,0x67,0x2f,0x31,0x39,0x39,0x39,0x2f,0x78, - 0x6c,0x69,0x6e,0x6b,0x22,0x20,0x78,0x3d,0x22,0x30, - 0x70,0x78,0x22,0x20,0x79,0x3d,0x22,0x30,0x70,0x78, - 0x22,0x0d,0x0a,0x09,0x20,0x77,0x69,0x64,0x74,0x68, - 0x3d,0x22,0x32,0x31,0x2e,0x37,0x37,0x35,0x70,0x78, - 0x22,0x20,0x68,0x65,0x69,0x67,0x68,0x74,0x3d,0x22, - 0x32,0x30,0x2e,0x37,0x36,0x32,0x70,0x78,0x22,0x20, - 0x76,0x69,0x65,0x77,0x42,0x6f,0x78,0x3d,0x22,0x30, - 0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37,0x35,0x20, - 0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x20,0x65,0x6e, - 0x61,0x62,0x6c,0x65,0x2d,0x62,0x61,0x63,0x6b,0x67, - 0x72,0x6f,0x75,0x6e,0x64,0x3d,0x22,0x6e,0x65,0x77, - 0x20,0x30,0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37, - 0x35,0x20,0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x20, - 0x78,0x6d,0x6c,0x3a,0x73,0x70,0x61,0x63,0x65,0x3d, - 0x22,0x70,0x72,0x65,0x73,0x65,0x72,0x76,0x65,0x22, - 0x3e,0x0d,0x0a,0x3c,0x70,0x61,0x74,0x68,0x20,0x66, - 0x69,0x6c,0x6c,0x3d,0x22,0x23,0x37,0x37,0x37,0x37, - 0x37,0x37,0x22,0x20,0x64,0x3d,0x22,0x4d,0x38,0x2e, - 0x37,0x31,0x2c,0x32,0x2e,0x34,0x38,0x63,0x30,0x2e, - 0x30,0x32,0x36,0x2c,0x30,0x2e,0x30,0x31,0x38,0x2c, - 0x30,0x2e,0x30,0x38,0x33,0x2c,0x30,0x2e,0x30,0x37, - 0x2c,0x30,0x2e,0x31,0x34,0x33,0x2c,0x30,0x2e,0x31, - 0x39,0x33,0x6c,0x31,0x2e,0x38,0x35,0x39,0x2c,0x33, - 0x2e,0x37,0x36,0x36,0x63,0x30,0x2e,0x33,0x33,0x33, - 0x2c,0x30,0x2e,0x36,0x37,0x36,0x2c,0x31,0x2e,0x31, - 0x38,0x2c,0x31,0x2e,0x32,0x39,0x31,0x2c,0x31,0x2e, - 0x39,0x32,0x34,0x2c,0x31,0x2e,0x33,0x39,0x38,0x6c, - 0x34,0x2e,0x31,0x35,0x35,0x2c,0x30,0x2e,0x36,0x30, - 0x33,0x0d,0x0a,0x09,0x63,0x30,0x2e,0x31,0x35,0x2c, - 0x30,0x2e,0x30,0x32,0x32,0x2c,0x30,0x2e,0x32,0x31, - 0x38,0x2c,0x30,0x2e,0x30,0x36,0x36,0x2c,0x30,0x2e, - 0x32,0x32,0x37,0x2c,0x30,0x2e,0x30,0x36,0x36,0x63, - 0x30,0x2e,0x30,0x30,0x31,0x2c,0x30,0x2c,0x30,0x2e, - 0x30,0x30,0x31,0x2c,0x30,0x2c,0x30,0x2e,0x30,0x30, - 0x31,0x2c,0x30,0x63,0x30,0x2c,0x30,0x2e,0x30,0x31, - 0x39,0x2d,0x30,0x2e,0x30,0x32,0x35,0x2c,0x30,0x2e, - 0x30,0x39,0x36,0x2d,0x30,0x2e,0x31,0x34,0x2c,0x30, - 0x2e,0x32,0x30,0x37,0x6c,0x2d,0x33,0x2e,0x30,0x30, - 0x39,0x2c,0x32,0x2e,0x39,0x33,0x31,0x0d,0x0a,0x09, - 0x63,0x2d,0x30,0x2e,0x35,0x33,0x39,0x2c,0x30,0x2e, - 0x35,0x32,0x36,0x2d,0x30,0x2e,0x38,0x36,0x32,0x2c, - 0x31,0x2e,0x35,0x32,0x31,0x2d,0x30,0x2e,0x37,0x33, - 0x35,0x2c,0x32,0x2e,0x32,0x36,0x33,0x6c,0x30,0x2e, - 0x37,0x31,0x31,0x2c,0x34,0x2e,0x31,0x34,0x63,0x30, + 0x55,0x54,0x46,0x2d,0x38,0x22,0x20,0x73,0x74,0x61, + 0x6e,0x64,0x61,0x6c,0x6f,0x6e,0x65,0x3d,0x22,0x6e, + 0x6f,0x22,0x3f,0x3e,0x0a,0x3c,0x21,0x2d,0x2d,0x20, + 0x47,0x65,0x6e,0x65,0x72,0x61,0x74,0x6f,0x72,0x3a, + 0x20,0x41,0x64,0x6f,0x62,0x65,0x20,0x49,0x6c,0x6c, + 0x75,0x73,0x74,0x72,0x61,0x74,0x6f,0x72,0x20,0x31, + 0x36,0x2e,0x30,0x2e,0x33,0x2c,0x20,0x53,0x56,0x47, + 0x20,0x45,0x78,0x70,0x6f,0x72,0x74,0x20,0x50,0x6c, + 0x75,0x67,0x2d,0x49,0x6e,0x20,0x2e,0x20,0x53,0x56, + 0x47,0x20,0x56,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3a, + 0x20,0x36,0x2e,0x30,0x30,0x20,0x42,0x75,0x69,0x6c, + 0x64,0x20,0x30,0x29,0x20,0x20,0x2d,0x2d,0x3e,0x0a, + 0x0a,0x3c,0x73,0x76,0x67,0x0a,0x20,0x20,0x20,0x78, + 0x6d,0x6c,0x6e,0x73,0x3a,0x64,0x63,0x3d,0x22,0x68, + 0x74,0x74,0x70,0x3a,0x2f,0x2f,0x70,0x75,0x72,0x6c, + 0x2e,0x6f,0x72,0x67,0x2f,0x64,0x63,0x2f,0x65,0x6c, + 0x65,0x6d,0x65,0x6e,0x74,0x73,0x2f,0x31,0x2e,0x31, + 0x2f,0x22,0x0a,0x20,0x20,0x20,0x78,0x6d,0x6c,0x6e, + 0x73,0x3a,0x63,0x63,0x3d,0x22,0x68,0x74,0x74,0x70, + 0x3a,0x2f,0x2f,0x63,0x72,0x65,0x61,0x74,0x69,0x76, + 0x65,0x63,0x6f,0x6d,0x6d,0x6f,0x6e,0x73,0x2e,0x6f, + 0x72,0x67,0x2f,0x6e,0x73,0x23,0x22,0x0a,0x20,0x20, + 0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x72,0x64,0x66, + 0x3d,0x22,0x68,0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77, + 0x77,0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f, + 0x31,0x39,0x39,0x39,0x2f,0x30,0x32,0x2f,0x32,0x32, + 0x2d,0x72,0x64,0x66,0x2d,0x73,0x79,0x6e,0x74,0x61, + 0x78,0x2d,0x6e,0x73,0x23,0x22,0x0a,0x20,0x20,0x20, + 0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x73,0x76,0x67,0x3d, + 0x22,0x68,0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77,0x77, + 0x77,0x2e,0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f,0x32, + 0x30,0x30,0x30,0x2f,0x73,0x76,0x67,0x22,0x0a,0x20, + 0x20,0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3d,0x22,0x68, + 0x74,0x74,0x70,0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e, + 0x77,0x33,0x2e,0x6f,0x72,0x67,0x2f,0x32,0x30,0x30, + 0x30,0x2f,0x73,0x76,0x67,0x22,0x0a,0x20,0x20,0x20, + 0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x73,0x6f,0x64,0x69, + 0x70,0x6f,0x64,0x69,0x3d,0x22,0x68,0x74,0x74,0x70, + 0x3a,0x2f,0x2f,0x73,0x6f,0x64,0x69,0x70,0x6f,0x64, + 0x69,0x2e,0x73,0x6f,0x75,0x72,0x63,0x65,0x66,0x6f, + 0x72,0x67,0x65,0x2e,0x6e,0x65,0x74,0x2f,0x44,0x54, + 0x44,0x2f,0x73,0x6f,0x64,0x69,0x70,0x6f,0x64,0x69, + 0x2d,0x30,0x2e,0x64,0x74,0x64,0x22,0x0a,0x20,0x20, + 0x20,0x78,0x6d,0x6c,0x6e,0x73,0x3a,0x69,0x6e,0x6b, + 0x73,0x63,0x61,0x70,0x65,0x3d,0x22,0x68,0x74,0x74, + 0x70,0x3a,0x2f,0x2f,0x77,0x77,0x77,0x2e,0x69,0x6e, + 0x6b,0x73,0x63,0x61,0x70,0x65,0x2e,0x6f,0x72,0x67, + 0x2f,0x6e,0x61,0x6d,0x65,0x73,0x70,0x61,0x63,0x65, + 0x73,0x2f,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x22,0x0a,0x20,0x20,0x20,0x76,0x65,0x72,0x73,0x69, + 0x6f,0x6e,0x3d,0x22,0x31,0x2e,0x31,0x22,0x0a,0x20, + 0x20,0x20,0x69,0x64,0x3d,0x22,0x45,0x62,0x65,0x6e, + 0x65,0x5f,0x31,0x22,0x0a,0x20,0x20,0x20,0x78,0x3d, + 0x22,0x30,0x70,0x78,0x22,0x0a,0x20,0x20,0x20,0x79, + 0x3d,0x22,0x30,0x70,0x78,0x22,0x0a,0x20,0x20,0x20, + 0x77,0x69,0x64,0x74,0x68,0x3d,0x22,0x32,0x31,0x2e, + 0x37,0x37,0x35,0x70,0x78,0x22,0x0a,0x20,0x20,0x20, + 0x68,0x65,0x69,0x67,0x68,0x74,0x3d,0x22,0x32,0x30, + 0x2e,0x37,0x36,0x32,0x70,0x78,0x22,0x0a,0x20,0x20, + 0x20,0x76,0x69,0x65,0x77,0x42,0x6f,0x78,0x3d,0x22, + 0x30,0x20,0x30,0x20,0x32,0x31,0x2e,0x37,0x37,0x35, + 0x20,0x32,0x30,0x2e,0x37,0x36,0x32,0x22,0x0a,0x20, + 0x20,0x20,0x65,0x6e,0x61,0x62,0x6c,0x65,0x2d,0x62, + 0x61,0x63,0x6b,0x67,0x72,0x6f,0x75,0x6e,0x64,0x3d, + 0x22,0x6e,0x65,0x77,0x20,0x30,0x20,0x30,0x20,0x32, + 0x31,0x2e,0x37,0x37,0x35,0x20,0x32,0x30,0x2e,0x37, + 0x36,0x32,0x22,0x0a,0x20,0x20,0x20,0x78,0x6d,0x6c, + 0x3a,0x73,0x70,0x61,0x63,0x65,0x3d,0x22,0x70,0x72, + 0x65,0x73,0x65,0x72,0x76,0x65,0x22,0x0a,0x20,0x20, + 0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a, + 0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3d,0x22,0x30, + 0x2e,0x39,0x31,0x20,0x72,0x31,0x33,0x37,0x32,0x35, + 0x22,0x0a,0x20,0x20,0x20,0x73,0x6f,0x64,0x69,0x70, + 0x6f,0x64,0x69,0x3a,0x64,0x6f,0x63,0x6e,0x61,0x6d, + 0x65,0x3d,0x22,0x73,0x74,0x61,0x72,0x5f,0x75,0x6e, + 0x66,0x69,0x6c,0x6c,0x65,0x64,0x2e,0x73,0x76,0x67, + 0x22,0x3e,0x3c,0x6d,0x65,0x74,0x61,0x64,0x61,0x74, + 0x61,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64,0x3d, + 0x22,0x6d,0x65,0x74,0x61,0x64,0x61,0x74,0x61,0x33, + 0x34,0x35,0x35,0x22,0x3e,0x3c,0x72,0x64,0x66,0x3a, + 0x52,0x44,0x46,0x3e,0x3c,0x63,0x63,0x3a,0x57,0x6f, + 0x72,0x6b,0x0a,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x72,0x64,0x66,0x3a,0x61,0x62,0x6f,0x75, + 0x74,0x3d,0x22,0x22,0x3e,0x3c,0x64,0x63,0x3a,0x66, + 0x6f,0x72,0x6d,0x61,0x74,0x3e,0x69,0x6d,0x61,0x67, + 0x65,0x2f,0x73,0x76,0x67,0x2b,0x78,0x6d,0x6c,0x3c, + 0x2f,0x64,0x63,0x3a,0x66,0x6f,0x72,0x6d,0x61,0x74, + 0x3e,0x3c,0x64,0x63,0x3a,0x74,0x79,0x70,0x65,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x72,0x64,0x66,0x3a,0x72,0x65,0x73,0x6f,0x75, + 0x72,0x63,0x65,0x3d,0x22,0x68,0x74,0x74,0x70,0x3a, + 0x2f,0x2f,0x70,0x75,0x72,0x6c,0x2e,0x6f,0x72,0x67, + 0x2f,0x64,0x63,0x2f,0x64,0x63,0x6d,0x69,0x74,0x79, + 0x70,0x65,0x2f,0x53,0x74,0x69,0x6c,0x6c,0x49,0x6d, + 0x61,0x67,0x65,0x22,0x20,0x2f,0x3e,0x3c,0x2f,0x63, + 0x63,0x3a,0x57,0x6f,0x72,0x6b,0x3e,0x3c,0x2f,0x72, + 0x64,0x66,0x3a,0x52,0x44,0x46,0x3e,0x3c,0x2f,0x6d, + 0x65,0x74,0x61,0x64,0x61,0x74,0x61,0x3e,0x3c,0x64, + 0x65,0x66,0x73,0x0a,0x20,0x20,0x20,0x20,0x20,0x69, + 0x64,0x3d,0x22,0x64,0x65,0x66,0x73,0x33,0x34,0x35, + 0x33,0x22,0x20,0x2f,0x3e,0x3c,0x73,0x6f,0x64,0x69, + 0x70,0x6f,0x64,0x69,0x3a,0x6e,0x61,0x6d,0x65,0x64, + 0x76,0x69,0x65,0x77,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x70,0x61,0x67,0x65,0x63,0x6f,0x6c,0x6f,0x72,0x3d, + 0x22,0x23,0x66,0x66,0x66,0x66,0x66,0x66,0x22,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x62,0x6f,0x72,0x64,0x65, + 0x72,0x63,0x6f,0x6c,0x6f,0x72,0x3d,0x22,0x23,0x36, + 0x36,0x36,0x36,0x36,0x36,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x62,0x6f,0x72,0x64,0x65,0x72,0x6f,0x70, + 0x61,0x63,0x69,0x74,0x79,0x3d,0x22,0x31,0x22,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x6f,0x62,0x6a,0x65,0x63, + 0x74,0x74,0x6f,0x6c,0x65,0x72,0x61,0x6e,0x63,0x65, + 0x3d,0x22,0x31,0x30,0x22,0x0a,0x20,0x20,0x20,0x20, + 0x20,0x67,0x72,0x69,0x64,0x74,0x6f,0x6c,0x65,0x72, + 0x61,0x6e,0x63,0x65,0x3d,0x22,0x31,0x30,0x22,0x0a, + 0x20,0x20,0x20,0x20,0x20,0x67,0x75,0x69,0x64,0x65, + 0x74,0x6f,0x6c,0x65,0x72,0x61,0x6e,0x63,0x65,0x3d, + 0x22,0x31,0x30,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x70, + 0x61,0x67,0x65,0x6f,0x70,0x61,0x63,0x69,0x74,0x79, + 0x3d,0x22,0x30,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x70, + 0x61,0x67,0x65,0x73,0x68,0x61,0x64,0x6f,0x77,0x3d, + 0x22,0x32,0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69, + 0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x77,0x69, + 0x6e,0x64,0x6f,0x77,0x2d,0x77,0x69,0x64,0x74,0x68, + 0x3d,0x22,0x36,0x34,0x30,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x3a,0x77,0x69,0x6e,0x64,0x6f,0x77,0x2d,0x68,0x65, + 0x69,0x67,0x68,0x74,0x3d,0x22,0x34,0x38,0x30,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64,0x3d,0x22, + 0x6e,0x61,0x6d,0x65,0x64,0x76,0x69,0x65,0x77,0x33, + 0x34,0x35,0x31,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x73,0x68,0x6f,0x77,0x67,0x72,0x69,0x64,0x3d,0x22, + 0x66,0x61,0x6c,0x73,0x65,0x22,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65, + 0x3a,0x7a,0x6f,0x6f,0x6d,0x3d,0x22,0x31,0x31,0x2e, + 0x33,0x36,0x36,0x39,0x32,0x31,0x22,0x0a,0x20,0x20, + 0x20,0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70, + 0x65,0x3a,0x63,0x78,0x3d,0x22,0x31,0x30,0x2e,0x38, + 0x38,0x37,0x35,0x22,0x0a,0x20,0x20,0x20,0x20,0x20, + 0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a,0x63, + 0x79,0x3d,0x22,0x31,0x30,0x2e,0x33,0x38,0x31,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e,0x6b,0x73, + 0x63,0x61,0x70,0x65,0x3a,0x77,0x69,0x6e,0x64,0x6f, + 0x77,0x2d,0x78,0x3d,0x22,0x36,0x39,0x22,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x69,0x6e,0x6b,0x73,0x63,0x61, + 0x70,0x65,0x3a,0x77,0x69,0x6e,0x64,0x6f,0x77,0x2d, + 0x79,0x3d,0x22,0x30,0x22,0x0a,0x20,0x20,0x20,0x20, + 0x20,0x69,0x6e,0x6b,0x73,0x63,0x61,0x70,0x65,0x3a, + 0x77,0x69,0x6e,0x64,0x6f,0x77,0x2d,0x6d,0x61,0x78, + 0x69,0x6d,0x69,0x7a,0x65,0x64,0x3d,0x22,0x30,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e,0x6b,0x73, + 0x63,0x61,0x70,0x65,0x3a,0x63,0x75,0x72,0x72,0x65, + 0x6e,0x74,0x2d,0x6c,0x61,0x79,0x65,0x72,0x3d,0x22, + 0x45,0x62,0x65,0x6e,0x65,0x5f,0x31,0x22,0x20,0x2f, + 0x3e,0x3c,0x70,0x61,0x74,0x68,0x0a,0x20,0x20,0x20, + 0x20,0x20,0x64,0x3d,0x22,0x6d,0x20,0x31,0x30,0x2e, + 0x39,0x35,0x39,0x39,0x39,0x38,0x2c,0x32,0x2e,0x34, + 0x38,0x20,0x63,0x20,0x30,0x2e,0x30,0x32,0x36,0x2c, + 0x30,0x2e,0x30,0x31,0x38,0x20,0x30,0x2e,0x30,0x38, + 0x33,0x2c,0x30,0x2e,0x30,0x37,0x20,0x30,0x2e,0x31, + 0x34,0x33,0x2c,0x30,0x2e,0x31,0x39,0x33,0x20,0x6c, + 0x20,0x31,0x2e,0x38,0x35,0x39,0x2c,0x33,0x2e,0x37, + 0x36,0x36,0x20,0x63,0x20,0x30,0x2e,0x33,0x33,0x33, + 0x2c,0x30,0x2e,0x36,0x37,0x36,0x20,0x31,0x2e,0x31, + 0x38,0x2c,0x31,0x2e,0x32,0x39,0x31,0x20,0x31,0x2e, + 0x39,0x32,0x34,0x2c,0x31,0x2e,0x33,0x39,0x38,0x20, + 0x6c,0x20,0x34,0x2e,0x31,0x35,0x35,0x2c,0x30,0x2e, + 0x36,0x30,0x33,0x20,0x63,0x20,0x30,0x2e,0x31,0x35, + 0x2c,0x30,0x2e,0x30,0x32,0x32,0x20,0x30,0x2e,0x32, + 0x31,0x38,0x2c,0x30,0x2e,0x30,0x36,0x36,0x20,0x30, + 0x2e,0x32,0x32,0x37,0x2c,0x30,0x2e,0x30,0x36,0x36, + 0x20,0x30,0x2e,0x30,0x30,0x31,0x2c,0x30,0x20,0x30, + 0x2e,0x30,0x30,0x31,0x2c,0x30,0x20,0x30,0x2e,0x30, + 0x30,0x31,0x2c,0x30,0x20,0x30,0x2c,0x30,0x2e,0x30, + 0x31,0x39,0x20,0x2d,0x30,0x2e,0x30,0x32,0x35,0x2c, + 0x30,0x2e,0x30,0x39,0x36,0x20,0x2d,0x30,0x2e,0x31, + 0x34,0x2c,0x30,0x2e,0x32,0x30,0x37,0x20,0x6c,0x20, + 0x2d,0x33,0x2e,0x30,0x30,0x39,0x2c,0x32,0x2e,0x39, + 0x33,0x31,0x20,0x63,0x20,0x2d,0x30,0x2e,0x35,0x33, + 0x39,0x2c,0x30,0x2e,0x35,0x32,0x36,0x20,0x2d,0x30, + 0x2e,0x38,0x36,0x32,0x2c,0x31,0x2e,0x35,0x32,0x31, + 0x20,0x2d,0x30,0x2e,0x37,0x33,0x35,0x2c,0x32,0x2e, + 0x32,0x36,0x33,0x20,0x6c,0x20,0x30,0x2e,0x37,0x31, + 0x31,0x2c,0x34,0x2e,0x31,0x34,0x20,0x63,0x20,0x30, 0x2e,0x30,0x32,0x37,0x2c,0x30,0x2e,0x31,0x35,0x36, - 0x2c,0x30,0x2e,0x30,0x30,0x33,0x2c,0x30,0x2e,0x32, - 0x33,0x34,0x2d,0x30,0x2e,0x30,0x31,0x31,0x2c,0x30, - 0x2e,0x32,0x33,0x39,0x63,0x2d,0x30,0x2e,0x30,0x32, - 0x35,0x2c,0x30,0x2d,0x30,0x2e,0x30,0x39,0x38,0x2d, - 0x30,0x2e,0x30,0x30,0x38,0x2d,0x30,0x2e,0x32,0x32, - 0x31,0x2d,0x30,0x2e,0x30,0x37,0x31,0x4c,0x39,0x2e, - 0x39,0x2c,0x31,0x36,0x2e,0x32,0x36,0x0d,0x0a,0x09, - 0x63,0x2d,0x30,0x2e,0x33,0x32,0x34,0x2d,0x30,0x2e, - 0x31,0x37,0x31,0x2d,0x30,0x2e,0x37,0x34,0x37,0x2d, - 0x30,0x2e,0x32,0x36,0x34,0x2d,0x31,0x2e,0x31,0x39, - 0x2d,0x30,0x2e,0x32,0x36,0x34,0x53,0x37,0x2e,0x38, - 0x34,0x33,0x2c,0x31,0x36,0x2e,0x30,0x39,0x2c,0x37, - 0x2e,0x35,0x32,0x2c,0x31,0x36,0x2e,0x32,0x36,0x6c, - 0x2d,0x33,0x2e,0x37,0x31,0x36,0x2c,0x31,0x2e,0x39, - 0x35,0x34,0x63,0x2d,0x30,0x2e,0x31,0x32,0x33,0x2c, - 0x30,0x2e,0x30,0x36,0x33,0x2d,0x30,0x2e,0x31,0x39, - 0x36,0x2c,0x30,0x2e,0x30,0x37,0x31,0x2d,0x30,0x2e, - 0x32,0x32,0x34,0x2c,0x30,0x2e,0x30,0x38,0x32,0x0d, - 0x0a,0x09,0x63,0x2d,0x30,0x2e,0x30,0x31,0x2d,0x30, - 0x2e,0x30,0x31,0x36,0x2d,0x30,0x2e,0x30,0x33,0x34, - 0x2d,0x30,0x2e,0x30,0x39,0x34,0x2d,0x30,0x2e,0x30, - 0x30,0x38,0x2d,0x30,0x2e,0x32,0x35,0x6c,0x30,0x2e, - 0x37,0x31,0x2d,0x34,0x2e,0x31,0x34,0x63,0x30,0x2e, - 0x31,0x32,0x38,0x2d,0x30,0x2e,0x37,0x34,0x33,0x2d, - 0x30,0x2e,0x31,0x39,0x35,0x2d,0x31,0x2e,0x37,0x33, - 0x37,0x2d,0x30,0x2e,0x37,0x33,0x35,0x2d,0x32,0x2e, - 0x32,0x36,0x33,0x4c,0x30,0x2e,0x35,0x34,0x31,0x2c, - 0x38,0x2e,0x37,0x31,0x32,0x43,0x30,0x2e,0x34,0x32, - 0x37,0x2c,0x38,0x2e,0x36,0x30,0x32,0x2c,0x30,0x2e, - 0x34,0x2c,0x38,0x2e,0x35,0x32,0x35,0x2c,0x30,0x2e, - 0x33,0x39,0x35,0x2c,0x38,0x2e,0x35,0x32,0x35,0x0d, - 0x0a,0x09,0x6c,0x30,0x2c,0x30,0x43,0x30,0x2e,0x34, - 0x30,0x35,0x2c,0x38,0x2e,0x35,0x31,0x2c,0x30,0x2e, - 0x34,0x37,0x32,0x2c,0x38,0x2e,0x34,0x36,0x34,0x2c, - 0x30,0x2e,0x36,0x33,0x2c,0x38,0x2e,0x34,0x34,0x31, - 0x6c,0x34,0x2e,0x31,0x35,0x33,0x2d,0x30,0x2e,0x36, - 0x30,0x33,0x43,0x35,0x2e,0x35,0x33,0x2c,0x37,0x2e, - 0x37,0x33,0x2c,0x36,0x2e,0x33,0x37,0x35,0x2c,0x37, - 0x2e,0x31,0x31,0x35,0x2c,0x36,0x2e,0x37,0x30,0x39, - 0x2c,0x36,0x2e,0x34,0x33,0x39,0x6c,0x31,0x2e,0x38, - 0x35,0x39,0x2d,0x33,0x2e,0x37,0x36,0x36,0x0d,0x0a, - 0x09,0x43,0x38,0x2e,0x36,0x32,0x38,0x2c,0x32,0x2e, - 0x35,0x35,0x2c,0x38,0x2e,0x36,0x38,0x35,0x2c,0x32, - 0x2e,0x34,0x39,0x36,0x2c,0x38,0x2e,0x37,0x31,0x2c, - 0x32,0x2e,0x34,0x38,0x20,0x4d,0x38,0x2e,0x37,0x31, - 0x2c,0x32,0x2e,0x30,0x37,0x36,0x63,0x2d,0x30,0x2e, - 0x31,0x38,0x32,0x2c,0x30,0x2d,0x30,0x2e,0x33,0x36, - 0x34,0x2c,0x30,0x2e,0x31,0x34,0x2d,0x30,0x2e,0x35, - 0x30,0x31,0x2c,0x30,0x2e,0x34,0x31,0x39,0x4c,0x36, - 0x2e,0x33,0x35,0x2c,0x36,0x2e,0x32,0x36,0x32,0x63, - 0x2d,0x30,0x2e,0x32,0x37,0x36,0x2c,0x30,0x2e,0x35, - 0x36,0x2d,0x31,0x2e,0x30,0x30,0x37,0x2c,0x31,0x2e, - 0x30,0x39,0x31,0x2d,0x31,0x2e,0x36,0x32,0x34,0x2c, - 0x31,0x2e,0x31,0x38,0x0d,0x0a,0x09,0x4c,0x30,0x2e, - 0x35,0x37,0x32,0x2c,0x38,0x2e,0x30,0x34,0x35,0x63, - 0x2d,0x30,0x2e,0x36,0x31,0x38,0x2c,0x30,0x2e,0x30, - 0x39,0x2d,0x30,0x2e,0x37,0x35,0x37,0x2c,0x30,0x2e, - 0x35,0x32,0x2d,0x30,0x2e,0x33,0x31,0x2c,0x30,0x2e, - 0x39,0x35,0x35,0x6c,0x33,0x2e,0x30,0x30,0x37,0x2c, - 0x32,0x2e,0x39,0x33,0x31,0x63,0x30,0x2e,0x34,0x34, - 0x37,0x2c,0x30,0x2e,0x34,0x33,0x36,0x2c,0x30,0x2e, - 0x37,0x32,0x36,0x2c,0x31,0x2e,0x32,0x39,0x35,0x2c, - 0x30,0x2e,0x36,0x32,0x2c,0x31,0x2e,0x39,0x30,0x39, - 0x6c,0x2d,0x30,0x2e,0x37,0x31,0x2c,0x34,0x2e,0x31, - 0x33,0x39,0x0d,0x0a,0x09,0x63,0x2d,0x30,0x2e,0x30, - 0x37,0x37,0x2c,0x30,0x2e,0x34,0x34,0x36,0x2c,0x30, + 0x20,0x30,0x2e,0x30,0x30,0x33,0x2c,0x30,0x2e,0x32, + 0x33,0x34,0x20,0x2d,0x30,0x2e,0x30,0x31,0x31,0x2c, + 0x30,0x2e,0x32,0x33,0x39,0x20,0x2d,0x30,0x2e,0x30, + 0x32,0x35,0x2c,0x30,0x20,0x2d,0x30,0x2e,0x30,0x39, + 0x38,0x2c,0x2d,0x30,0x2e,0x30,0x30,0x38,0x20,0x2d, + 0x30,0x2e,0x32,0x32,0x31,0x2c,0x2d,0x30,0x2e,0x30, + 0x37,0x31,0x20,0x6c,0x20,0x2d,0x33,0x2e,0x37,0x31, + 0x34,0x2c,0x2d,0x31,0x2e,0x39,0x35,0x35,0x20,0x63, + 0x20,0x2d,0x30,0x2e,0x33,0x32,0x34,0x2c,0x2d,0x30, + 0x2e,0x31,0x37,0x31,0x20,0x2d,0x30,0x2e,0x37,0x34, + 0x37,0x2c,0x2d,0x30,0x2e,0x32,0x36,0x34,0x20,0x2d, + 0x31,0x2e,0x31,0x39,0x2c,0x2d,0x30,0x2e,0x32,0x36, + 0x34,0x20,0x2d,0x30,0x2e,0x34,0x34,0x33,0x2c,0x30, + 0x20,0x2d,0x30,0x2e,0x38,0x36,0x37,0x2c,0x30,0x2e, + 0x30,0x39,0x34,0x20,0x2d,0x31,0x2e,0x31,0x38,0x39, + 0x39,0x39,0x39,0x38,0x2c,0x30,0x2e,0x32,0x36,0x34, + 0x20,0x6c,0x20,0x2d,0x33,0x2e,0x37,0x31,0x36,0x2c, + 0x31,0x2e,0x39,0x35,0x34,0x20,0x63,0x20,0x2d,0x30, + 0x2e,0x31,0x32,0x33,0x2c,0x30,0x2e,0x30,0x36,0x33, + 0x20,0x2d,0x30,0x2e,0x31,0x39,0x36,0x2c,0x30,0x2e, + 0x30,0x37,0x31,0x20,0x2d,0x30,0x2e,0x32,0x32,0x34, + 0x2c,0x30,0x2e,0x30,0x38,0x32,0x20,0x2d,0x30,0x2e, + 0x30,0x31,0x2c,0x2d,0x30,0x2e,0x30,0x31,0x36,0x20, + 0x2d,0x30,0x2e,0x30,0x33,0x34,0x2c,0x2d,0x30,0x2e, + 0x30,0x39,0x34,0x20,0x2d,0x30,0x2e,0x30,0x30,0x38, + 0x2c,0x2d,0x30,0x2e,0x32,0x35,0x20,0x6c,0x20,0x30, + 0x2e,0x37,0x31,0x2c,0x2d,0x34,0x2e,0x31,0x34,0x20, + 0x63,0x20,0x30,0x2e,0x31,0x32,0x38,0x2c,0x2d,0x30, + 0x2e,0x37,0x34,0x33,0x20,0x2d,0x30,0x2e,0x31,0x39, + 0x35,0x2c,0x2d,0x31,0x2e,0x37,0x33,0x37,0x20,0x2d, + 0x30,0x2e,0x37,0x33,0x35,0x2c,0x2d,0x32,0x2e,0x32, + 0x36,0x33,0x20,0x6c,0x20,0x2d,0x33,0x2e,0x30,0x30, + 0x36,0x2c,0x2d,0x32,0x2e,0x39,0x33,0x31,0x20,0x63, + 0x20,0x2d,0x30,0x2e,0x31,0x31,0x34,0x2c,0x2d,0x30, + 0x2e,0x31,0x31,0x20,0x2d,0x30,0x2e,0x31,0x34,0x31, + 0x2c,0x2d,0x30,0x2e,0x31,0x38,0x37,0x20,0x2d,0x30, + 0x2e,0x31,0x34,0x36,0x2c,0x2d,0x30,0x2e,0x31,0x38, + 0x37,0x20,0x6c,0x20,0x30,0x2c,0x30,0x20,0x63,0x20, + 0x30,0x2e,0x30,0x31,0x2c,0x2d,0x30,0x2e,0x30,0x31, + 0x35,0x20,0x30,0x2e,0x30,0x37,0x37,0x2c,0x2d,0x30, + 0x2e,0x30,0x36,0x31,0x20,0x30,0x2e,0x32,0x33,0x35, + 0x2c,0x2d,0x30,0x2e,0x30,0x38,0x34,0x20,0x6c,0x20, + 0x34,0x2e,0x31,0x35,0x33,0x2c,0x2d,0x30,0x2e,0x36, + 0x30,0x33,0x20,0x63,0x20,0x30,0x2e,0x37,0x34,0x37, + 0x2c,0x2d,0x30,0x2e,0x31,0x30,0x38,0x20,0x31,0x2e, + 0x35,0x39,0x32,0x2c,0x2d,0x30,0x2e,0x37,0x32,0x33, + 0x20,0x31,0x2e,0x39,0x32,0x36,0x2c,0x2d,0x31,0x2e, + 0x33,0x39,0x39,0x20,0x4c,0x20,0x31,0x30,0x2e,0x38, + 0x31,0x37,0x39,0x39,0x38,0x2c,0x32,0x2e,0x36,0x37, + 0x33,0x20,0x63,0x20,0x30,0x2e,0x30,0x36,0x2c,0x2d, + 0x30,0x2e,0x31,0x32,0x33,0x20,0x30,0x2e,0x31,0x31, + 0x37,0x2c,0x2d,0x30,0x2e,0x31,0x37,0x37,0x20,0x30, + 0x2e,0x31,0x34,0x32,0x2c,0x2d,0x30,0x2e,0x31,0x39, + 0x33,0x20,0x6d,0x20,0x30,0x2c,0x2d,0x30,0x2e,0x34, + 0x30,0x34,0x20,0x63,0x20,0x2d,0x30,0x2e,0x31,0x38, + 0x32,0x2c,0x30,0x20,0x2d,0x30,0x2e,0x33,0x36,0x34, + 0x2c,0x30,0x2e,0x31,0x34,0x20,0x2d,0x30,0x2e,0x35, + 0x30,0x31,0x2c,0x30,0x2e,0x34,0x31,0x39,0x20,0x4c, + 0x20,0x38,0x2e,0x35,0x39,0x39,0x39,0x39,0x38,0x32, + 0x2c,0x36,0x2e,0x32,0x36,0x32,0x20,0x63,0x20,0x2d, + 0x30,0x2e,0x32,0x37,0x36,0x2c,0x30,0x2e,0x35,0x36, + 0x20,0x2d,0x31,0x2e,0x30,0x30,0x37,0x2c,0x31,0x2e, + 0x30,0x39,0x31,0x20,0x2d,0x31,0x2e,0x36,0x32,0x34, + 0x2c,0x31,0x2e,0x31,0x38,0x20,0x6c,0x20,0x2d,0x34, + 0x2e,0x31,0x35,0x34,0x2c,0x30,0x2e,0x36,0x30,0x33, + 0x20,0x63,0x20,0x2d,0x30,0x2e,0x36,0x31,0x38,0x2c, + 0x30,0x2e,0x30,0x39,0x20,0x2d,0x30,0x2e,0x37,0x35, + 0x37,0x2c,0x30,0x2e,0x35,0x32,0x20,0x2d,0x30,0x2e, + 0x33,0x31,0x2c,0x30,0x2e,0x39,0x35,0x35,0x20,0x6c, + 0x20,0x33,0x2e,0x30,0x30,0x37,0x2c,0x32,0x2e,0x39, + 0x33,0x31,0x20,0x63,0x20,0x30,0x2e,0x34,0x34,0x37, + 0x2c,0x30,0x2e,0x34,0x33,0x36,0x20,0x30,0x2e,0x37, + 0x32,0x36,0x2c,0x31,0x2e,0x32,0x39,0x35,0x20,0x30, + 0x2e,0x36,0x32,0x2c,0x31,0x2e,0x39,0x30,0x39,0x20, + 0x6c,0x20,0x2d,0x30,0x2e,0x37,0x31,0x2c,0x34,0x2e, + 0x31,0x33,0x39,0x20,0x63,0x20,0x2d,0x30,0x2e,0x30, + 0x37,0x37,0x2c,0x30,0x2e,0x34,0x34,0x36,0x20,0x30, 0x2e,0x30,0x39,0x34,0x2c,0x30,0x2e,0x37,0x30,0x38, - 0x2c,0x30,0x2e,0x34,0x30,0x35,0x2c,0x30,0x2e,0x37, - 0x30,0x38,0x63,0x30,0x2e,0x31,0x31,0x38,0x2c,0x30, - 0x2c,0x30,0x2e,0x32,0x35,0x35,0x2d,0x30,0x2e,0x30, - 0x33,0x38,0x2c,0x30,0x2e,0x34,0x30,0x37,0x2d,0x30, - 0x2e,0x31,0x31,0x37,0x6c,0x33,0x2e,0x37,0x31,0x37, - 0x2d,0x31,0x2e,0x39,0x35,0x35,0x63,0x30,0x2e,0x32, - 0x37,0x36,0x2d,0x30,0x2e,0x31,0x34,0x35,0x2c,0x30, - 0x2e,0x36,0x34,0x2d,0x30,0x2e,0x32,0x31,0x36,0x2c, - 0x31,0x2e,0x30,0x30,0x33,0x2d,0x30,0x2e,0x32,0x31, - 0x36,0x0d,0x0a,0x09,0x63,0x30,0x2e,0x33,0x36,0x34, - 0x2c,0x30,0x2c,0x30,0x2e,0x37,0x32,0x39,0x2c,0x30, - 0x2e,0x30,0x37,0x32,0x2c,0x31,0x2e,0x30,0x30,0x34, - 0x2c,0x30,0x2e,0x32,0x31,0x36,0x6c,0x33,0x2e,0x37, - 0x31,0x37,0x2c,0x31,0x2e,0x39,0x35,0x35,0x63,0x30, - 0x2e,0x31,0x35,0x2c,0x30,0x2e,0x30,0x37,0x39,0x2c, - 0x30,0x2e,0x32,0x38,0x39,0x2c,0x30,0x2e,0x31,0x31, - 0x37,0x2c,0x30,0x2e,0x34,0x30,0x37,0x2c,0x30,0x2e, - 0x31,0x31,0x37,0x63,0x30,0x2e,0x33,0x31,0x31,0x2c, - 0x30,0x2c,0x30,0x2e,0x34,0x38,0x31,0x2d,0x30,0x2e, - 0x32,0x36,0x32,0x2c,0x30,0x2e,0x34,0x30,0x35,0x2d, - 0x30,0x2e,0x37,0x30,0x38,0x6c,0x2d,0x30,0x2e,0x37, - 0x31,0x31,0x2d,0x34,0x2e,0x31,0x33,0x39,0x0d,0x0a, - 0x09,0x63,0x2d,0x30,0x2e,0x31,0x30,0x35,0x2d,0x30, - 0x2e,0x36,0x31,0x34,0x2c,0x30,0x2e,0x31,0x37,0x34, - 0x2d,0x31,0x2e,0x34,0x37,0x34,0x2c,0x30,0x2e,0x36, - 0x31,0x39,0x2d,0x31,0x2e,0x39,0x30,0x39,0x6c,0x33, - 0x2e,0x30,0x30,0x39,0x2d,0x32,0x2e,0x39,0x33,0x31, - 0x63,0x30,0x2e,0x34,0x34,0x36,0x2d,0x30,0x2e,0x34, - 0x33,0x35,0x2c,0x30,0x2e,0x33,0x30,0x37,0x2d,0x30, - 0x2e,0x38,0x36,0x35,0x2d,0x30,0x2e,0x33,0x31,0x31, - 0x2d,0x30,0x2e,0x39,0x35,0x35,0x6c,0x2d,0x34,0x2e, - 0x31,0x35,0x34,0x2d,0x30,0x2e,0x36,0x30,0x33,0x0d, - 0x0a,0x09,0x63,0x2d,0x30,0x2e,0x36,0x31,0x37,0x2d, - 0x30,0x2e,0x30,0x38,0x39,0x2d,0x31,0x2e,0x33,0x34, - 0x39,0x2d,0x30,0x2e,0x36,0x32,0x2d,0x31,0x2e,0x36, - 0x32,0x34,0x2d,0x31,0x2e,0x31,0x38,0x4c,0x39,0x2e, - 0x32,0x31,0x33,0x2c,0x32,0x2e,0x34,0x39,0x35,0x43, - 0x39,0x2e,0x30,0x37,0x34,0x2c,0x32,0x2e,0x32,0x31, - 0x36,0x2c,0x38,0x2e,0x38,0x39,0x33,0x2c,0x32,0x2e, - 0x30,0x37,0x36,0x2c,0x38,0x2e,0x37,0x31,0x2c,0x32, - 0x2e,0x30,0x37,0x36,0x4c,0x38,0x2e,0x37,0x31,0x2c, - 0x32,0x2e,0x30,0x37,0x36,0x7a,0x22,0x2f,0x3e,0x0d, - 0x0a,0x3c,0x2f,0x73,0x76,0x67,0x3e,0x0d,0x0a + 0x20,0x30,0x2e,0x34,0x30,0x35,0x2c,0x30,0x2e,0x37, + 0x30,0x38,0x20,0x30,0x2e,0x31,0x31,0x38,0x2c,0x30, + 0x20,0x30,0x2e,0x32,0x35,0x35,0x2c,0x2d,0x30,0x2e, + 0x30,0x33,0x38,0x20,0x30,0x2e,0x34,0x30,0x37,0x2c, + 0x2d,0x30,0x2e,0x31,0x31,0x37,0x20,0x6c,0x20,0x33, + 0x2e,0x37,0x31,0x37,0x2c,0x2d,0x31,0x2e,0x39,0x35, + 0x35,0x20,0x63,0x20,0x30,0x2e,0x32,0x37,0x35,0x39, + 0x39,0x39,0x38,0x2c,0x2d,0x30,0x2e,0x31,0x34,0x35, + 0x20,0x30,0x2e,0x36,0x33,0x39,0x39,0x39,0x39,0x38, + 0x2c,0x2d,0x30,0x2e,0x32,0x31,0x36,0x20,0x31,0x2e, + 0x30,0x30,0x32,0x39,0x39,0x39,0x38,0x2c,0x2d,0x30, + 0x2e,0x32,0x31,0x36,0x20,0x30,0x2e,0x33,0x36,0x34, + 0x2c,0x30,0x20,0x30,0x2e,0x37,0x32,0x39,0x2c,0x30, + 0x2e,0x30,0x37,0x32,0x20,0x31,0x2e,0x30,0x30,0x34, + 0x2c,0x30,0x2e,0x32,0x31,0x36,0x20,0x6c,0x20,0x33, + 0x2e,0x37,0x31,0x37,0x2c,0x31,0x2e,0x39,0x35,0x35, + 0x20,0x63,0x20,0x30,0x2e,0x31,0x35,0x2c,0x30,0x2e, + 0x30,0x37,0x39,0x20,0x30,0x2e,0x32,0x38,0x39,0x2c, + 0x30,0x2e,0x31,0x31,0x37,0x20,0x30,0x2e,0x34,0x30, + 0x37,0x2c,0x30,0x2e,0x31,0x31,0x37,0x20,0x30,0x2e, + 0x33,0x31,0x31,0x2c,0x30,0x20,0x30,0x2e,0x34,0x38, + 0x31,0x2c,0x2d,0x30,0x2e,0x32,0x36,0x32,0x20,0x30, + 0x2e,0x34,0x30,0x35,0x2c,0x2d,0x30,0x2e,0x37,0x30, + 0x38,0x20,0x6c,0x20,0x2d,0x30,0x2e,0x37,0x31,0x31, + 0x2c,0x2d,0x34,0x2e,0x31,0x33,0x39,0x20,0x63,0x20, + 0x2d,0x30,0x2e,0x31,0x30,0x35,0x2c,0x2d,0x30,0x2e, + 0x36,0x31,0x34,0x20,0x30,0x2e,0x31,0x37,0x34,0x2c, + 0x2d,0x31,0x2e,0x34,0x37,0x34,0x20,0x30,0x2e,0x36, + 0x31,0x39,0x2c,0x2d,0x31,0x2e,0x39,0x30,0x39,0x20, + 0x4c,0x20,0x31,0x39,0x2e,0x34,0x31,0x30,0x39,0x39, + 0x38,0x2c,0x39,0x20,0x63,0x20,0x30,0x2e,0x34,0x34, + 0x36,0x2c,0x2d,0x30,0x2e,0x34,0x33,0x35,0x20,0x30, + 0x2e,0x33,0x30,0x37,0x2c,0x2d,0x30,0x2e,0x38,0x36, + 0x35,0x20,0x2d,0x30,0x2e,0x33,0x31,0x31,0x2c,0x2d, + 0x30,0x2e,0x39,0x35,0x35,0x20,0x6c,0x20,0x2d,0x34, + 0x2e,0x31,0x35,0x34,0x2c,0x2d,0x30,0x2e,0x36,0x30, + 0x33,0x20,0x63,0x20,0x2d,0x30,0x2e,0x36,0x31,0x37, + 0x2c,0x2d,0x30,0x2e,0x30,0x38,0x39,0x20,0x2d,0x31, + 0x2e,0x33,0x34,0x39,0x2c,0x2d,0x30,0x2e,0x36,0x32, + 0x20,0x2d,0x31,0x2e,0x36,0x32,0x34,0x2c,0x2d,0x31, + 0x2e,0x31,0x38,0x20,0x6c,0x20,0x2d,0x31,0x2e,0x38, + 0x35,0x39,0x2c,0x2d,0x33,0x2e,0x37,0x36,0x37,0x20, + 0x63,0x20,0x2d,0x30,0x2e,0x31,0x33,0x39,0x2c,0x2d, + 0x30,0x2e,0x32,0x37,0x39,0x20,0x2d,0x30,0x2e,0x33, + 0x32,0x2c,0x2d,0x30,0x2e,0x34,0x31,0x39,0x20,0x2d, + 0x30,0x2e,0x35,0x30,0x33,0x2c,0x2d,0x30,0x2e,0x34, + 0x31,0x39,0x20,0x6c,0x20,0x30,0x2c,0x30,0x20,0x7a, + 0x22,0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x64,0x3d, + 0x22,0x70,0x61,0x74,0x68,0x33,0x34,0x34,0x39,0x22, + 0x0a,0x20,0x20,0x20,0x20,0x20,0x69,0x6e,0x6b,0x73, + 0x63,0x61,0x70,0x65,0x3a,0x63,0x6f,0x6e,0x6e,0x65, + 0x63,0x74,0x6f,0x72,0x2d,0x63,0x75,0x72,0x76,0x61, + 0x74,0x75,0x72,0x65,0x3d,0x22,0x30,0x22,0x0a,0x20, + 0x20,0x20,0x20,0x20,0x73,0x74,0x79,0x6c,0x65,0x3d, + 0x22,0x66,0x69,0x6c,0x6c,0x3a,0x23,0x37,0x37,0x37, + 0x37,0x37,0x37,0x22,0x20,0x2f,0x3e,0x3c,0x2f,0x73, + 0x76,0x67,0x3e }; + diff --git a/data/resources/star_filled.svg b/data/resources/star_filled.svg index 11d7e0f411..520dfb585d 100644 --- a/data/resources/star_filled.svg +++ b/data/resources/star_filled.svg @@ -1,12 +1,49 @@ - + - - - - + +image/svg+xml \ No newline at end of file diff --git a/data/resources/star_unfilled.svg b/data/resources/star_unfilled.svg index d1063e40ae..f123f6d6c3 100644 --- a/data/resources/star_unfilled.svg +++ b/data/resources/star_unfilled.svg @@ -1,18 +1,49 @@ - + - - - - + +image/svg+xml \ No newline at end of file diff --git a/es-app/CMakeLists.txt b/es-app/CMakeLists.txt index d2dec02224..b872c243de 100644 --- a/es-app/CMakeLists.txt +++ b/es-app/CMakeLists.txt @@ -25,6 +25,7 @@ set(ES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.h ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.h ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiRefreshDatabase.h # Scrapers ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h @@ -69,6 +70,7 @@ set(ES_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiRefreshDatabase.cpp # Scrapers ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp diff --git a/es-app/src/EmulationStation.h b/es-app/src/EmulationStation.h index 3f166fb42a..3fbb279050 100644 --- a/es-app/src/EmulationStation.h +++ b/es-app/src/EmulationStation.h @@ -5,7 +5,7 @@ #define PROGRAM_VERSION_MAJOR 2 #define PROGRAM_VERSION_MINOR 0 #define PROGRAM_VERSION_MAINTENANCE 1 -#define PROGRAM_VERSION_STRING "2.0.1" +#define PROGRAM_VERSION_STRING "2.0.1a" #define PROGRAM_BUILT_STRING __DATE__ " - " __TIME__ diff --git a/es-app/src/FileData.cpp b/es-app/src/FileData.cpp index 350966aa57..095e5136a9 100644 --- a/es-app/src/FileData.cpp +++ b/es-app/src/FileData.cpp @@ -5,6 +5,21 @@ namespace fs = boost::filesystem; +MetaDataListType fileTypeToMetaDataType(FileType type) +{ + switch(type) + { + case GAME: + return GAME_METADATA; + case FOLDER: + return FOLDER_METADATA; + case FILTER: + return FILTER_METADATA; + } + + return GAME_METADATA; +} + std::string removeParenthesis(const std::string& str) { // remove anything in parenthesis or brackets @@ -51,16 +66,17 @@ std::string getCleanGameName(const std::string& str, const SystemData* system) if(system && system->hasPlatformId(PlatformIds::ARCADE) || system->hasPlatformId(PlatformIds::NEOGEO)) stem = PlatformIds::getCleanMameName(stem.c_str()); - return removeParenthesis(stem); + return stem; } FileData::FileData(const std::string& fileID, SystemData* system, FileType type, const std::string& nameCache) - : mFileID(fileID), mSystem(system), mType(type), mNameCache(nameCache) + : mFileID(fileID), mSystem(system), mType(type), mNameCache(nameCache), mMetaDataCache(fileTypeToMetaDataType(type), false) { } FileData::FileData() : FileData("", NULL, (FileType)0) { + } FileData::FileData(const std::string& fileID, const std::string& systemID, FileType type) : @@ -98,11 +114,17 @@ FileType FileData::getType() const MetaDataMap FileData::get_metadata() const { - return SystemManager::getInstance()->database().getFileData(mFileID, mSystem->getName()); + if(mValidMetaDataCache) return mMetaDataCache; + mMetaDataCache = SystemManager::getInstance()->database().getFileData(mFileID, mSystem->getName()); + mValidMetaDataCache = true; + return mMetaDataCache; } -void FileData::set_metadata(const MetaDataMap& metadata) +void FileData::set_metadata(const MetaDataMap& metadata) const { + // The database round trip might alter some values. + // Setting metadata should be fairly infrequent, so this should be fine. + mValidMetaDataCache = false; SystemManager::getInstance()->database().setFileData(mFileID, getSystemID(), mType, metadata); } @@ -111,7 +133,23 @@ std::vector FileData::getChildren(const FileSort* sort) const if(sort == NULL) sort = &getFileSorts().at(Settings::getInstance()->getInt("SortTypeIndex")); - return SystemManager::getInstance()->database().getChildrenOf(mFileID, mSystem, true, true, sort); + bool foldersFirst = Settings::getInstance()->getBool("SortFoldersFirst"); + + if(mType == FILTER) + { + MetaDataMap metadata = get_metadata(); + //Sorry about abusing columns for other purposes. + std::string filter_matches = mMetaDataCache.get("genre"); + int limit = mMetaDataCache.get("players"); + //TODO: Let a filter also specify ability to match filters/folders + std::string orderby = mMetaDataCache.get("developer"); + FileSort filterSort("Filter given sort",orderby.c_str()); + if(!orderby.empty()) + sort = &filterSort; + return SystemManager::getInstance()->database().getChildrenOfFilter(mFileID, mSystem, false, filter_matches, limit, foldersFirst, sort); + } + + return SystemManager::getInstance()->database().getChildrenOf(mFileID, mSystem, true, true, foldersFirst, sort); } std::vector FileData::getChildrenRecursive(bool includeFolders, const FileSort* sort) const @@ -119,5 +157,9 @@ std::vector FileData::getChildrenRecursive(bool includeFolders, const if(sort == NULL) sort = &getFileSorts().at(Settings::getInstance()->getInt("SortTypeIndex")); - return SystemManager::getInstance()->database().getChildrenOf(mFileID, mSystem, false, includeFolders, sort); + bool foldersFirst = Settings::getInstance()->getBool("SortFoldersFirst"); + + if(mType == FILTER) + return getChildren(sort); + return SystemManager::getInstance()->database().getChildrenOf(mFileID, mSystem, false, includeFolders, foldersFirst, sort); } diff --git a/es-app/src/FileData.h b/es-app/src/FileData.h index 46eaf8c8b5..171f056107 100644 --- a/es-app/src/FileData.h +++ b/es-app/src/FileData.h @@ -11,13 +11,16 @@ struct FileSort; enum FileType { GAME = 1, - FOLDER = 2 + FOLDER = 2, + FILTER = 3 }; // Used for loading/saving gamelist.xml. const char* fileTypeToString(FileType type); FileType stringToFileType(const char* str); +MetaDataListType fileTypeToMetaDataType(FileType type); + // Remove (.*) and [.*] from str std::string removeParenthesis(const std::string& str); @@ -34,7 +37,7 @@ class FileData inline bool operator!=(const FileData& rhs) const { return !(*this == rhs); } MetaDataMap get_metadata() const; - void set_metadata(const MetaDataMap& metadata); + void set_metadata(const MetaDataMap& metadata) const; const std::string& getName() const; FileType getType() const; @@ -56,4 +59,6 @@ class FileData FileType mType; mutable std::string mNameCache; + mutable MetaDataMap mMetaDataCache; + mutable bool mValidMetaDataCache = false; }; diff --git a/es-app/src/GamelistDB.cpp b/es-app/src/GamelistDB.cpp index 395272b409..abe62a5815 100644 --- a/es-app/src/GamelistDB.cpp +++ b/es-app/src/GamelistDB.cpp @@ -5,6 +5,7 @@ #include #include #include +#include "SystemManager.h" namespace fs = boost::filesystem; @@ -29,23 +30,22 @@ fs::path fileIDToPath(const std::string& fileID, const SystemData* system) return resolvePath(fileID, system->getStartPath(), true); } -MetaDataListType fileTypeToMetaDataType(FileType type) -{ - switch(type) - { - case GAME: - return GAME_METADATA; - case FOLDER: - return FOLDER_METADATA; - } - assert(false); - return GAME_METADATA; -} std::vector sFileSorts = boost::assign::list_of - (FileSort("Alphabetical, asc", "ORDER BY name")) - (FileSort("Alphabetical, desc", "ORDER BY name DESC")); + (FileSort("Alphabetical", "LOWER(name)")) + (FileSort("Reverse Alphabetical", "LOWER(name) DESC")) + (FileSort("Highest Rating", "rating DESC, LOWER(name)")) + (FileSort("Lowest Rating", "rating, LOWER(name)")) + (FileSort("Lowest Rating (Non-zero)", "case when rating = 0 then 1 else 0 end, rating, LOWER(name)")) + (FileSort("Chronological", "releasedate IS NULL, releasedate, LOWER(name)")) + (FileSort("Reverse Chronological", "releasedate DESC, LOWER(name)")) + (FileSort("Played Recently", "lastplayed DESC, LOWER(name)")) + (FileSort("Played Long Ago", "lastplayed IS NULL, lastplayed, LOWER(name)")) + (FileSort("Most Played", "playcount DESC, LOWER(name)")) + (FileSort("Least Played", "playcount, LOWER(name)")) + (FileSort("Least Played (Non-zero)", "case when playcount = 0 then 1 else 0 end, playcount, LOWER(name)")) + (FileSort("Random", "RANDOM()")); const std::vector& getFileSorts() { @@ -289,6 +289,12 @@ void GamelistDB::createMissingTables() if(sqlite3_exec(mDB, ss.str().c_str(), NULL, NULL, NULL)) throw DBException() << "Error creating table!\n\t" << sqlite3_errmsg(mDB); + + if(sqlite3_exec(mDB, "CREATE TABLE IF NOT EXISTS tagtable (fileid VARCHAR(255) NOT NULL, systemid VARCHAR(255) NOT NULL, tag VARCHAR(255) NOT NULL, PRIMARY KEY (fileid, systemid, tag), FOREIGN KEY(fileid,systemid) REFERENCES files(fileid,systemid))", NULL, NULL, NULL)) + throw DBException() << "Error creating table!\n\t" << sqlite3_errmsg(mDB); + sqlite3_exec(mDB, "DROP VIEW tags", NULL, NULL, NULL); + if(sqlite3_exec(mDB, "CREATE VIEW IF NOT EXISTS tags (tag) as select tag from tagtable where tagtable.fileid = files.fileid and tagtable.systemid = files.systemid", NULL, NULL, NULL)) + throw DBException() << "Error creating table!\n\t" << sqlite3_errmsg(mDB); } // returns a vector of all columns in a particular table @@ -393,7 +399,7 @@ void add_file(const char* fileid, FileType filetype, const SystemData* system, s std::string clean_name = getCleanGameName(fileid, system); if(sqlite3_bind_text(insert_stmt, 3, clean_name.c_str(), clean_name.size(), SQLITE_STATIC)) - throw DBException() << "Error binding filetype in populate().\n\t" << sqlite3_errmsg(db); + throw DBException() << "Error binding name in populate().\n\t" << sqlite3_errmsg(db); if(sqlite3_step(insert_stmt) != SQLITE_DONE) throw DBException() << "Error adding file \"" << fileid << "\" in populate().\n\t" << sqlite3_errmsg(db); @@ -472,9 +478,17 @@ void GamelistDB::addMissingFiles(const SystemData* system) SQLTransaction transaction(mDB); - // actually start adding things - populate_recursive(relativeTo, extensions, relativeTo, system, mDB, stmt); - + // actually start adding things. + // go ahead and add the root always. + sqlite3_bind_text(stmt,1,".",1,SQLITE_STATIC); + sqlite3_bind_int(stmt,2,FileType::FOLDER); + sqlite3_bind_null(stmt,3); + stmt.step_expected(SQLITE_DONE); + sqlite3_reset(stmt); + if(!relativeTo.empty()) + { + populate_recursive(relativeTo, extensions, relativeTo, system, mDB, stmt); + } transaction.commit(); } @@ -482,7 +496,7 @@ void GamelistDB::updateExists(const SystemData* system) { const std::string& relativeTo = system->getStartPath(); - SQLPreparedStmt readStmt(mDB, "SELECT fileid FROM files WHERE systemid = ?1"); + SQLPreparedStmt readStmt(mDB, "SELECT fileid,fileexists,filetype FROM files WHERE systemid = ?1"); sqlite3_bind_text(readStmt, 1, system->getName().c_str(), system->getName().size(), SQLITE_STATIC); SQLPreparedStmt updateStmt(mDB, "UPDATE files SET fileexists = ?1 WHERE fileid = ?2 AND systemid = ?3"); @@ -490,20 +504,21 @@ void GamelistDB::updateExists(const SystemData* system) SQLTransaction transaction(mDB); - // for each game, check if its file exists - if it doesn't remove it from the DB + // for each game, check if its file exists while(readStmt.step() != SQLITE_DONE) { const char* path = (const char*)sqlite3_column_text(readStmt, 0); + bool existsOld = sqlite3_column_int(readStmt, 1) > 0; - bool exists = false; + bool existsNew = false; if(path && path[0] == '.') // it's relative - exists = fs::exists(relativeTo + "/" + path); + existsNew = fs::exists(relativeTo + "/" + path); else - exists = fs::exists(path); + existsNew = fs::exists(path); sqlite3_bind_text(updateStmt, 2, path, strlen(path), SQLITE_STATIC); - sqlite3_bind_int(updateStmt, 1, exists); - updateStmt.step_expected(SQLITE_DONE); + sqlite3_bind_int(updateStmt, 1, existsNew); + if(existsNew != existsOld) updateStmt.step_expected(SQLITE_DONE); updateStmt.reset(); } @@ -514,12 +529,34 @@ void GamelistDB::updateExists(const FileData& file) { SQLPreparedStmt stmt(mDB, "UPDATE files SET fileexists = ?1 WHERE fileid = ?2 AND systemid = ?3"); bool exists = fs::exists(fileIDToPath(file.getFileID(), file.getSystem())); + if(file.getType() == FILTER) exists = true; sqlite3_bind_int(stmt, 1, exists); sqlite3_bind_text(stmt, 2, file.getFileID().c_str(), file.getFileID().size(), SQLITE_STATIC); sqlite3_bind_text(stmt, 3, file.getSystem()->getName().c_str(), file.getSystem()->getName().size(), SQLITE_STATIC); stmt.step_expected(SQLITE_DONE); } +void GamelistDB::removeEntry(const FileData& file) +{ + SQLPreparedStmt stmt(mDB, "DELETE FROM files WHERE fileid = ?1 AND systemid = ?2"); + bool exists = fs::exists(fileIDToPath(file.getFileID(), file.getSystem())); + sqlite3_bind_text(stmt, 1, file.getFileID().c_str(), file.getFileID().size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, file.getSystem()->getName().c_str(), file.getSystem()->getName().size(), SQLITE_STATIC); + stmt.step_expected(SQLITE_DONE); +} + +//Remove rows where the fileexists flag is not set. +//No check to verify the truth of that flag is performed. +void GamelistDB::removeNonexisting(const SystemData* system) +{ + SQLPreparedStmt deleteStmt(mDB, "DELETE FROM files WHERE systemid = ?1 AND fileexists = '0'"); + sqlite3_bind_text(deleteStmt, 1, system->getName().c_str(), system->getName().size(), SQLITE_STATIC); + + SQLTransaction transaction(mDB); + deleteStmt.step_expected(SQLITE_DONE); + transaction.commit(); +} + MetaDataMap GamelistDB::getFileData(const std::string& fileID, const std::string& systemID) const { SQLPreparedStmt readStmt(mDB, "SELECT * FROM files WHERE fileid = ?1 AND systemid = ?2"); @@ -550,7 +587,7 @@ void GamelistDB::setFileData(const std::string& fileID, const std::string& syste std::stringstream ss; ss << "INSERT OR REPLACE INTO files VALUES (?1, ?2, ?3, ?4, "; - auto& mdd = metadata.getMDD(); + const std::vector& mdd = getMDDMap().at(GAME_METADATA); for(unsigned int i = 0; i < mdd.size(); i++) { ss << "?" << i + RESERVED_COLUMNS + 1; @@ -570,32 +607,91 @@ void GamelistDB::setFileData(const std::string& fileID, const std::string& syste for(unsigned int i = 0; i < mdd.size(); i++) { const std::string& val = metadata.get(mdd.at(i).key); - sqlite3_bind_text(stmt, i + RESERVED_COLUMNS + 1, val.c_str(), val.size(), SQLITE_STATIC); + if((mdd.at(i).type == MD_TIME || mdd.at(i).type == MD_DATE) && (val == "not-a-date-time" || val.empty())) + { + sqlite3_bind_null(stmt, i + RESERVED_COLUMNS + 1); + }else{ + sqlite3_bind_text(stmt, i + RESERVED_COLUMNS + 1, val.c_str(), val.size(), SQLITE_STATIC); + } } stmt.step_expected(SQLITE_DONE); } + +std::vector GamelistDB::getFileTags(const std::string& fileID, const std::string& systemID) const +{ + SQLPreparedStmt stmt(mDB, "SELECT tag FROM tagtable WHERE fileid = ?1 AND systemid = ?2"); + sqlite3_bind_text(stmt, 1, fileID.c_str(), fileID.size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, systemID.c_str(), systemID.size(), SQLITE_STATIC); + + std::vector tags; + + while(stmt.step() != SQLITE_DONE) + { + tags.push_back((const char*)sqlite3_column_text(stmt,0)); + } + + return tags; +} + + +void GamelistDB::setFileTag(const std::string& fileID, const std::string& systemID, const std::string& tagID, bool value) +{ + if(value) + { + SQLPreparedStmt stmt(mDB, "INSERT OR IGNORE INTO tagtable VALUES ( ?1, ?2, ?3 )"); + sqlite3_bind_text(stmt, 1, fileID.c_str(), fileID.size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, systemID.c_str(), systemID.size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, tagID.c_str(), tagID.size(), SQLITE_STATIC); + + SQLTransaction transaction(mDB); + stmt.step_expected(SQLITE_DONE); + transaction.commit(); + }else{ + SQLPreparedStmt stmt(mDB, "DELETE FROM tagtable where fileid = ?1 AND systemid = ?2 AND tag = ?3"); + sqlite3_bind_text(stmt, 1, fileID.c_str(), fileID.size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, systemID.c_str(), systemID.size(), SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, tagID.c_str(), tagID.size(), SQLITE_STATIC); + + SQLTransaction transaction(mDB); + stmt.step_expected(SQLITE_DONE); + transaction.commit(); + } +} + std::vector GamelistDB::getChildrenOf(const std::string& fileID, SystemData* system, - bool immediateChildrenOnly, bool includeFolders, const FileSort* sortType) + bool immediateChildrenOnly, bool includeFolders, bool foldersFirst, const FileSort* sortType) { const std::string& systemID = system->getName(); + const std::string& systemFilter = system->getFilterQuery(); + const std::string& systemPath = system->getStartPath(); std::vector children; std::stringstream ss; - ss << "SELECT fileid, name, filetype FROM files WHERE systemid = ?1 AND fileexists = 1 "; + ss << "SELECT fileid,systemid,name,filetype FROM files WHERE 1 "; + if(!systemFilter.empty()) + { + ss << "AND (" << systemFilter << ") "; + } + if(!system->isMetaSystem()) + { + ss << "AND ( systemid = ?1 ) "; + } if(immediateChildrenOnly) ss << "AND inimmediatedir(fileid, ?2) "; else ss << "AND indir(fileid, ?2) "; if(!includeFolders) - ss << "AND NOT filetype = " << FileType::FOLDER << " "; + ss << " AND NOT filetype in ( " << FileType::FOLDER << "," << FileType::FILTER << ") "; + ss << " ORDER BY "; + if(includeFolders && foldersFirst) ss << " CASE WHEN filetype = 1 THEN 1 ELSE 0 END, "; if(sortType) ss << sortType->sql; else - ss << "ORDER BY name"; + ss << "LOWER(name)"; std::string query = ss.str(); SQLPreparedStmt stmt(mDB, query); @@ -605,14 +701,121 @@ std::vector GamelistDB::getChildrenOf(const std::string& fileID, Syste while(stmt.step() != SQLITE_DONE) { const char* fileid = (const char*)sqlite3_column_text(stmt, 0); - const char* name = (const char*)sqlite3_column_text(stmt, 1); - FileType filetype = (FileType)sqlite3_column_int(stmt, 2); - children.push_back(FileData(fileid, system, filetype, name ? name : "")); + std::string systemname((const char*)sqlite3_column_text(stmt,1)); + SystemData* childSystem = (systemname == systemID) ? system : SystemManager::getInstance()->getSystemByName(systemname); + const char* name = (const char*)sqlite3_column_text(stmt, 2); + FileType filetype = (FileType)sqlite3_column_int(stmt, 3); + if(childSystem) + children.push_back(FileData(fileid, childSystem, filetype, name ? name : "")); } return children; } +std::vector GamelistDB::getChildrenOfFilter(const std::string& fileID, SystemData* system, + bool matchFolders, const std::string& filter_matches, int limit, bool foldersFirst, const FileSort* sortType) +{ + const std::string& systemID = system->getName(); + const std::string& systemFilter = system->getFilterQuery(); + const std::string& systemPath = system->getStartPath(); + std::vector children; + + std::stringstream ss; + //Use the indir logic to support filters having subfilters + //A subfilter may return more entries than the parent! + //The user can handle making sure subfilters make subsets. + ss << "SELECT fileid, systemid, name, filetype, CAST(strftime(\"%Y\",releasedate) as INTEGER) as year FROM files WHERE "; + + + ss << "( (inimmediatedir(fileid, ?2) AND systemid = ?1) OR ( "; + + if(!systemFilter.empty()) + ss << " (" << systemFilter << ") AND "; + if(!system->isMetaSystem()) + ss << " ( systemid = ?1 ) AND "; + if(!filter_matches.empty()) + { + ss << " (" << filter_matches << ") "; + }else{ + //Just match everything, put something to make the OR syntax work. + ss << " 1 "; + } + if(!matchFolders) + ss << " AND filetype is " << FileType::GAME; + ss << " )) "; + + ss << " ORDER BY "; + if(foldersFirst) ss << " CASE WHEN filetype = 1 THEN 1 ELSE 0 END, "; + if(sortType) + ss << sortType->sql; + else + ss << "LOWER(name)"; + if(limit > 0) + ss << " LIMIT ?3"; + + std::string query = ss.str(); + LOG(LogDebug) << "(" << fileID << ","< 0) + sqlite3_bind_int(stmt, 3, limit); + + while(stmt.step() != SQLITE_DONE) + { + const char* fileid = (const char*)sqlite3_column_text(stmt, 0); + std::string systemname((const char*)sqlite3_column_text(stmt,1)); + SystemData* childSystem = (systemname == systemID) ? system : SystemManager::getInstance()->getSystemByName(systemname); + const char* name = (const char*)sqlite3_column_text(stmt, 2); + FileType filetype = (FileType)sqlite3_column_int(stmt, 3); + if (childSystem != NULL) + children.push_back(FileData(fileid, childSystem, filetype, name ? name : "")); + } + }catch(...){ + //statement failed, likely to do to syntax in the filter. + //we'll just return the empty children vector. + } + + return children; +} +bool GamelistDB::systemHasFileWithImage(const SystemData* system) +{ + const std::string& systemID = system->getName(); + const std::string& systemFilter = system->getFilterQuery(); + const std::string& systemPath = system->getStartPath(); + std::stringstream ss; + ss << "SELECT EXISTS(SELECT 1 FROM files WHERE image IS NOT NULL AND image <> '' "; + if(!systemFilter.empty()) + ss << "AND (" << systemFilter << ") "; + if(!system->isMetaSystem()) + ss << "AND ( systemid = ?1 ) "; + ss << ")"; + SQLPreparedStmt readStmt(mDB, ss.str()); + if(!system->isMetaSystem()) sqlite3_bind_text(readStmt, 1, systemID.c_str(), systemID.size(), SQLITE_STATIC); + + readStmt.step_expected(SQLITE_ROW); + return sqlite3_column_int(readStmt,0); +} +int GamelistDB::getSystemFileCount(const SystemData* system) +{ + const std::string& systemID = system->getName(); + const std::string& systemFilter = system->getFilterQuery(); + const std::string& systemPath = system->getStartPath(); + std::stringstream ss; + ss << "SELECT COUNT(1) FROM files WHERE filetype = 1 "; + if(!systemFilter.empty()) + ss << "AND (" << systemFilter << ") "; + if(!system->isMetaSystem()) + ss << "AND ( systemid = ?1 ) "; + SQLPreparedStmt readStmt(mDB, ss.str()); + if(!system->isMetaSystem()) sqlite3_bind_text(readStmt, 1, systemID.c_str(), systemID.size(), SQLITE_STATIC); + + readStmt.step_expected(SQLITE_ROW); + return sqlite3_column_int(readStmt,0); +} void GamelistDB::importXML(const SystemData* system, const std::string& xml_path) { LOG(LogInfo) << "Appending gamelist.xml file \"" << xml_path << "\" to database (system: " << system->getName() << ")..."; @@ -735,3 +938,8 @@ void GamelistDB::exportXML(const SystemData* system, const std::string& xml_path doc.save_file(xml_path.c_str()); } + +int GamelistDB::totalChanges() +{ + return sqlite3_total_changes(mDB); +} \ No newline at end of file diff --git a/es-app/src/GamelistDB.h b/es-app/src/GamelistDB.h index 648f391707..868cfbc781 100644 --- a/es-app/src/GamelistDB.h +++ b/es-app/src/GamelistDB.h @@ -47,21 +47,40 @@ class GamelistDB void addMissingFiles(const SystemData* system); void updateExists(const SystemData* system); void updateExists(const FileData& file); // update the fileexists flag for a particular file + void removeEntry(const FileData& file); + void removeNonexisting(const SystemData* system); // returns all metadata for a given fileID MetaDataMap getFileData(const std::string& fileID, const std::string& systemID) const; + // tags are a separate notion from meta data + std::vector getFileTags(const std::string& fileID, const std::string& systemID) const; // Sets all metadata for a given fileID (overwrites existing data). void setFileData(const std::string& fileID, const std::string& systemID, FileType type, const MetaDataMap& metadata); + // If value is true, add the tag. If the value is false, remove it. + void setFileTag(const std::string& fileID, const std::string& systemID, const std::string& tagID, bool value); // returns either all immediate children (immediateChildrenOnly) OR // all children, children of children, etc. of file ID (immedateChildrenOnly = false) std::vector getChildrenOf(const std::string& fileID, SystemData* system, - bool immediateChildrenOnly, bool includeFolders, const FileSort* sortType = NULL); - + bool immediateChildrenOnly, bool includeFolders, bool foldersFirst, const FileSort* sortType = NULL); + + // reads and runs a filter from the database. + // filters support a filesystem like hierarchy reusing a lot of the code + // from getChildrenOf(). + // A filters are opaque to getting children. + //(You'll need to call this again to get "grandchildren") + std::vector getChildrenOfFilter(const std::string& fileID, SystemData* system, + bool matchFolders, const std::string& filter_matches, int limit, bool foldersFirst, const FileSort* sortType); + + bool systemHasFileWithImage(const SystemData* system); + int getSystemFileCount(const SystemData* system); void importXML(const SystemData* system, const std::string& xml_path); void exportXML(const SystemData* system, const std::string& xml_path); + //Return the total_changes value from sqlite. Allows basic UI awareness. + int totalChanges(); + private: void openDB(const char* path); void createMissingTables(); // will do nothing if a "files" table already exists diff --git a/es-app/src/MetaData.cpp b/es-app/src/MetaData.cpp index 507e97ef35..90fd6895d1 100644 --- a/es-app/src/MetaData.cpp +++ b/es-app/src/MetaData.cpp @@ -14,22 +14,36 @@ MetaDataDecl gameDecls[] = { {"genre", MD_STRING, "unknown", false, "genre", "enter game genre"}, {"players", MD_INT, "1", false, "players", "enter number of players"}, {"playcount", MD_INT, "0", true, "play count", "enter number of times played"}, - {"lastplayed", MD_TIME, "0", true, "last played", "enter last played date"} + {"lastplayed", MD_TIME, "not-a-date-time", true, "last played", "enter last played date"} }; // because of how the GamelistDB is set up, this must be a subset of gameDecls MetaDataDecl folderDecls[] = { - {"name", MD_STRING, "", false}, - {"desc", MD_MULTILINE_STRING, "", false}, - {"image", MD_IMAGE_PATH, "", false}, - {"thumbnail", MD_IMAGE_PATH, "", false}, + {"name", MD_STRING, "", false, "name", "enter game name"}, + {"desc", MD_MULTILINE_STRING, "", false, "description", "enter description"}, + {"image", MD_IMAGE_PATH, "", false, "image", "enter path to image"}, + {"thumbnail", MD_IMAGE_PATH, "", false, "thumbnail", "enter path to thumbnail"}, +}; +// because of that subset constraint, note the abuse of the fields! +// Some are marked as MD_MULTILINE_STRING instead of MD_STRING. This appears to be ok. +MetaDataDecl filterDecls[] = { + {"name", MD_STRING, "Filter", false, "name", "enter filter name"}, + {"desc", MD_MULTILINE_STRING, "", false, "description", "enter description"}, + {"image", MD_IMAGE_PATH, "", false, "image", "enter path to image"}, + {"thumbnail", MD_IMAGE_PATH, "", false, "thumbnail", "enter path to thumbnail"}, + {"genre", MD_MULTILINE_STRING, "rating > .6 AND playcount > 0", false, "query", "enter query"}, + {"developer", MD_STRING, "", false, "order by", "enter columns to order by"}, + {"players", MD_INT, "0", false, "limit", "enter limit on results"} }; std::map< MetaDataListType, std::vector > MDD_map = boost::assign::map_list_of (GAME_METADATA, std::vector(gameDecls, gameDecls + sizeof(gameDecls) / sizeof(gameDecls[0]))) (FOLDER_METADATA, - std::vector(folderDecls, folderDecls + sizeof(folderDecls) / sizeof(folderDecls[0]))); + std::vector(folderDecls, folderDecls + sizeof(folderDecls) / sizeof(folderDecls[0]))) + (FILTER_METADATA, + std::vector(filterDecls, filterDecls + sizeof(filterDecls) / sizeof(filterDecls[0]))); + const std::map >& getMDDMap() { return MDD_map; @@ -38,7 +52,23 @@ const std::map >& getMDDMap() MetaDataMap::MetaDataMap(MetaDataListType type) : mType(type) { + setDefaults(); +} +MetaDataMap::MetaDataMap(MetaDataListType type, bool init) + : mType(type) +{ + if(init) setDefaults(); +} + +void MetaDataMap::setDefaults() +{ + //To enforce that subset constraint above, we intialize the map to have + //all the defaults of the gameDecls, with our specific defaults overriding. + const std::vector& mddGame = getMDDMap().at(GAME_METADATA); const std::vector& mdd = getMDD(); + for(auto iter = mddGame.begin(); iter != mddGame.end(); iter++) + set(iter->key, iter->defaultValue); + if(mType == GAME_METADATA) return; for(auto iter = mdd.begin(); iter != mdd.end(); iter++) set(iter->key, iter->defaultValue); } diff --git a/es-app/src/MetaData.h b/es-app/src/MetaData.h index f9bbd381bd..34187adab4 100644 --- a/es-app/src/MetaData.h +++ b/es-app/src/MetaData.h @@ -36,7 +36,8 @@ struct MetaDataDecl enum MetaDataListType { GAME_METADATA, - FOLDER_METADATA + FOLDER_METADATA, + FILTER_METADATA, }; const std::map< MetaDataListType, std::vector >& getMDDMap(); @@ -46,6 +47,7 @@ class MetaDataMap { public: MetaDataMap(MetaDataListType type); + MetaDataMap(MetaDataListType type, bool initialize); inline MetaDataListType getType() const { return mType; } inline const std::vector& getMDD() const { return getMDDMap().at(getType()); } @@ -81,6 +83,8 @@ class MetaDataMap set(key.c_str(), value); } + void setDefaults(); + private: MetaDataListType mType; std::map mMap; @@ -95,6 +99,6 @@ inline boost::posix_time::ptime MetaDataMap::get(const char* key) const template<> inline void MetaDataMap::set(const char* key, const boost::posix_time::ptime& time) { - mMap[key] = boost::posix_time::to_iso_string(time); + mMap[key] = ptime_to_string(time, SQLITE_TIME_STRING_FORMAT); } diff --git a/es-app/src/PlatformId.cpp b/es-app/src/PlatformId.cpp index 2242396d34..492df1e634 100644 --- a/es-app/src/PlatformId.cpp +++ b/es-app/src/PlatformId.cpp @@ -62,6 +62,7 @@ namespace PlatformIds "wonderswan", "wonderswancolor", "zxspectrum", + "virtualboy", "ignore", // do not allow scraping for this system "invalid" diff --git a/es-app/src/PlatformId.h b/es-app/src/PlatformId.h index 59c2156448..a1dc11e5a3 100644 --- a/es-app/src/PlatformId.h +++ b/es-app/src/PlatformId.h @@ -62,6 +62,7 @@ namespace PlatformIds WONDERSWAN, WONDERSWAN_COLOR, ZX_SPECTRUM, + VIRTUAL_BOY, PLATFORM_IGNORE, // do not allow scraping for this system PLATFORM_COUNT diff --git a/es-app/src/SystemData.cpp b/es-app/src/SystemData.cpp index 512e004880..7922459637 100644 --- a/es-app/src/SystemData.cpp +++ b/es-app/src/SystemData.cpp @@ -8,13 +8,14 @@ #include "VolumeControl.h" #include "Log.h" #include "InputManager.h" +#include "SystemManager.h" #include #include "Settings.h" namespace fs = boost::filesystem; SystemData::SystemData(const std::string& name, const std::string& fullName, const std::string& startPath, const std::vector& extensions, - const std::string& command, const std::vector& platformIds, const std::string& themeFolder) : + const std::string& command, const std::vector& platformIds, const std::string& themeFolder, const std::string& filterQuery) : mRoot(FileData(".", this, FileType::FOLDER)) { mName = name; @@ -32,6 +33,19 @@ SystemData::SystemData(const std::string& name, const std::string& fullName, con mLaunchCommand = command; mPlatformIds = platformIds; mThemeFolder = themeFolder; + mFilterQuery = filterQuery; + MetaDataMap md = MetaDataMap(FOLDER_METADATA,true); + if(mStartPath.empty()) + { + mRoot = FileData(".", this, FileType::FILTER); + md = MetaDataMap(FILTER_METADATA,true); + //TODO: Fix schema to avoid this column abuse. + md.set("genre",""); + md.set("players",0); + md.set("developer",""); + } + //User might have switched this system from a pure filter to a folder, so we need to update the DB. + mRoot.set_metadata(md); loadTheme(); } @@ -83,7 +97,7 @@ std::string escapePath(const boost::filesystem::path& path) #endif } -void SystemData::launchGame(Window* window, FileData game) const +void SystemData::launchGame(Window* window, const FileData& game) const { LOG(LogInfo) << "Attempting to launch game..."; @@ -126,6 +140,7 @@ void SystemData::launchGame(Window* window, FileData game) const metadata.set("lastplayed", time); game.set_metadata(metadata); + } std::string SystemData::getThemePath() const @@ -145,7 +160,7 @@ std::string SystemData::getThemePath() const unsigned int SystemData::getGameCount() const { - return getRootFolder().getChildrenRecursive(false).size(); + return SystemManager::getInstance()->database().getSystemFileCount(this); } void SystemData::loadTheme() @@ -169,13 +184,5 @@ void SystemData::loadTheme() bool SystemData::hasFileWithImage() const { - // TODO: optimize this with an SQL query - std::vector files = getRootFolder().getChildrenRecursive(true); - for(auto it = files.begin(); it != files.end(); it++) - { - if(!it->get_metadata().get("image").empty()) - return true; - } - - return false; + return SystemManager::getInstance()->database().systemHasFileWithImage(this); } diff --git a/es-app/src/SystemData.h b/es-app/src/SystemData.h index 3320fb9f2a..587459d29a 100644 --- a/es-app/src/SystemData.h +++ b/es-app/src/SystemData.h @@ -12,7 +12,7 @@ class SystemData { public: SystemData(const std::string& name, const std::string& fullName, const std::string& startPath, const std::vector& extensions, - const std::string& command, const std::vector& platformIds, const std::string& themeFolder); + const std::string& command, const std::vector& platformIds, const std::string& themeFolder, const std::string& filterQuery); virtual ~SystemData(); inline const std::string& getName() const { return mName; } @@ -23,6 +23,8 @@ class SystemData inline bool hasPlatformId(PlatformIds::PlatformId id) const { return std::find(mPlatformIds.begin(), mPlatformIds.end(), id) != mPlatformIds.end(); } inline const std::string& getThemeFolder() const { return mThemeFolder; } inline const std::shared_ptr& getTheme() const { return mTheme; } + inline const std::string& getFilterQuery() const { return mFilterQuery; } + inline bool isMetaSystem() const { return mStartPath.empty(); } inline const FileData& getRootFolder() const { return mRoot; } @@ -31,7 +33,7 @@ class SystemData unsigned int getGameCount() const; bool hasFileWithImage() const; - void launchGame(Window* window, FileData game) const; + void launchGame(Window* window, const FileData& game) const; // Load or re-load theme. void loadTheme(); @@ -45,6 +47,7 @@ class SystemData std::vector mPlatformIds; std::string mThemeFolder; std::shared_ptr mTheme; + std::string mFilterQuery; FileData mRoot; }; diff --git a/es-app/src/SystemManager.cpp b/es-app/src/SystemManager.cpp index 5476c19de9..b3d0fbeb3c 100644 --- a/es-app/src/SystemManager.cpp +++ b/es-app/src/SystemManager.cpp @@ -84,12 +84,13 @@ void SystemManager::loadConfig() // load each system in the config file for(pugi::xml_node system = systemList.child("system"); system; system = system.next_sibling("system")) { - std::string name, fullname, path, cmd, themeFolder; + std::string name, fullname, path, cmd, themeFolder, filter; PlatformIds::PlatformId platformId = PlatformIds::PLATFORM_UNKNOWN; name = system.child("name").text().get(); fullname = system.child("fullname").text().get(); path = system.child("path").text().get(); + filter = system.child("filter").text().get(); // convert extensions list from a string into a vector of strings std::vector extensions = readList(system.child("extension").text().get()); @@ -124,7 +125,10 @@ void SystemManager::loadConfig() themeFolder = system.child("theme").text().as_string(name.c_str()); // validate as best we can (make sure we're not missing required information) - if(!isValidSystemName(name) || path.empty() || extensions.empty() || cmd.empty()) + // If we have a filter, we don't need a path, extensions, or cmd. + if(!isValidSystemName(name)) + throw ESException() << "System \"" << name << "\" has an invalid or missing name!"; + if(filter.empty() && ((path.empty() || extensions.empty() || cmd.empty()))) throw ESException() << "System \"" << name << "\" is missing name, path, extension, or command!"; // convert path to generic directory seperators @@ -135,20 +139,30 @@ void SystemManager::loadConfig() if(path[path.size() - 1] == '/') path = path.erase(path.size() - 1); - SystemData* newSys = new SystemData(name, fullname, path, extensions, cmd, platformIds, themeFolder); - mDatabase.addMissingFiles(newSys); - mDatabase.updateExists(newSys); + //empty path means this is a filter system. Scraping does not make sense for these. + if(path.empty()) + platformIds.push_back(PlatformIds::PLATFORM_IGNORE); + + SystemData* newSys = new SystemData(name, fullname, path, extensions, cmd, platformIds, themeFolder, filter); if(newSys->getGameCount() == 0) { - LOG(LogWarning) << "System \"" << name << "\" has no games! Ignoring it."; - delete newSys; + LOG(LogWarning) << "System \"" << name << "\" has no games! Running file system scan."; + mDatabase.addMissingFiles(newSys); + if(newSys->getGameCount() == 0) + { + LOG(LogWarning) << "System \"" << name << "\" still has no games! Ignoring it."; + + delete newSys; + }else{ + mSystems.push_back(newSys); + } + }else{ mSystems.push_back(newSys); } } - updateDatabase(); } void SystemManager::writeExampleConfig(const fs::path& path) @@ -189,6 +203,12 @@ void SystemManager::writeExampleConfig(const fs::path& path) " \n" " nes\n" + "\n" + " \n" + " \n" " \n" "\n"; diff --git a/es-app/src/animations/MoveCameraAnimation.h b/es-app/src/animations/MoveCameraAnimation.h index 1f1689bb29..ca8255e0dd 100644 --- a/es-app/src/animations/MoveCameraAnimation.h +++ b/es-app/src/animations/MoveCameraAnimation.h @@ -11,6 +11,7 @@ class MoveCameraAnimation : public Animation void apply(float t) override { + cameraOut = Eigen::Affine3f::Identity(); // cubic ease out t -= 1; cameraOut.translation() = -lerp(-mCameraStart.translation(), mTarget, t*t*t + 1); diff --git a/es-app/src/components/RatingComponent.cpp b/es-app/src/components/RatingComponent.cpp index 12f39a3090..d3b1989367 100644 --- a/es-app/src/components/RatingComponent.cpp +++ b/es-app/src/components/RatingComponent.cpp @@ -25,12 +25,14 @@ void RatingComponent::setValue(const std::string& value) else if(mValue < 0.0f) mValue = 0.0f; } + mValueGiven = value; updateVertices(); } std::string RatingComponent::getValue() const { + if (!mValueGiven.empty()) return mValueGiven; //We didn't change anything, so avoid introducing rounding // do not use std::to_string here as it will use the current locale // and that sometimes encodes decimals as commas std::stringstream ss; @@ -129,9 +131,13 @@ bool RatingComponent::input(InputConfig* config, Input input) { if(config->isMappedTo("a", input) && input.value != 0) { - mValue += 1.f / NUM_RATING_STARS; - if(mValue > 1.0f) + mValue += 1.f / (NUM_RATING_STARS * 2); + if(mValue > (1.0f + .5f/(NUM_RATING_STARS * 2))) mValue = 0.0f; + if(mValue > 1.0f && mValue < (1.0f + .6f/(NUM_RATING_STARS * 2))) + mValue = 1.0f; + + mValueGiven.clear(); //invalidate the cached value; updateVertices(); } @@ -168,6 +174,6 @@ void RatingComponent::applyTheme(const std::shared_ptr& theme, const std::vector RatingComponent::getHelpPrompts() { std::vector prompts; - prompts.push_back(HelpPrompt("a", "add star")); + prompts.push_back(HelpPrompt("a", "add half star")); return prompts; } diff --git a/es-app/src/components/RatingComponent.h b/es-app/src/components/RatingComponent.h index a275c2701e..fd8eed3fa5 100644 --- a/es-app/src/components/RatingComponent.h +++ b/es-app/src/components/RatingComponent.h @@ -31,6 +31,7 @@ class RatingComponent : public GuiComponent void updateVertices(); float mValue; + std::string mValueGiven; //Cache the exact string, so we can give it back unchanged. struct Vertex { diff --git a/es-app/src/components/TextListComponent.h b/es-app/src/components/TextListComponent.h index 8067cd7391..f561c8dec2 100644 --- a/es-app/src/components/TextListComponent.h +++ b/es-app/src/components/TextListComponent.h @@ -124,7 +124,7 @@ TextListComponent::TextListComponent(Window* window) : mColors[0] = 0x0000FFFF; mColors[1] = 0x00FF00FF; } - +#include template void TextListComponent::render(const Eigen::Affine3f& parentTrans) { @@ -151,17 +151,25 @@ void TextListComponent::render(const Eigen::Affine3f& parentTrans) startEntry = size() - screenCount; } + float overflow = (mCursor - startEntry)*entrySize + (entrySize + font->getHeight())/2 - mSize.y(); float y = 0; + + if (overflow > 0) + { + y = - overflow; + } int listCutoff = startEntry + screenCount; if(listCutoff > size()) listCutoff = size(); + + // draw selector bar if(startEntry < listCutoff) { Renderer::setMatrix(trans); - Renderer::drawRect(0.f, (mCursor - startEntry)*entrySize + (entrySize - font->getHeight())/2, mSize.x(), font->getHeight(), mSelectorColor); + Renderer::drawRect(0.f, y + (mCursor - startEntry)*entrySize + (entrySize - font->getHeight())/2, mSize.x(), font->getHeight(), mSelectorColor); } // clip to inside margins @@ -217,6 +225,7 @@ void TextListComponent::render(const Eigen::Affine3f& parentTrans) y += entrySize; } + //pop margin clip Renderer::popClipRect(); listRenderTitleOverlay(trans); diff --git a/es-app/src/guis/GuiGamelistOptions.cpp b/es-app/src/guis/GuiGamelistOptions.cpp index 0839a3540e..0ae83ea8c2 100644 --- a/es-app/src/guis/GuiGamelistOptions.cpp +++ b/es-app/src/guis/GuiGamelistOptions.cpp @@ -1,42 +1,35 @@ #include "GuiGamelistOptions.h" #include "GuiMetaDataEd.h" +#include "GuiMenu.h" +#include "Window.h" +#include "Sound.h" +#include "Log.h" +#include "Settings.h" +#include "guis/GuiMsgBox.h" +#include "guis/GuiSettings.h" #include "views/gamelist/IGameListView.h" #include "views/ViewController.h" +#include "components/ButtonComponent.h" +#include "components/SwitchComponent.h" +#include "components/SliderComponent.h" +#include "components/TextComponent.h" +#include "components/OptionListComponent.h" +#include "components/MenuComponent.h" +#include "guis/GuiTextEditPopup.h" +#include "VolumeControl.h" #include "SystemManager.h" #include "Settings.h" GuiGamelistOptions::GuiGamelistOptions(Window* window, SystemData* system) : GuiComponent(window), mSystem(system), - mMenu(window, "OPTIONS") + mMenu(window, "MENU") { addChild(&mMenu); - - // jump to letter - char curChar = toupper(getGamelist()->getCursor().getName()[0]); - if(curChar < 'A' || curChar > 'Z') // in the case of unicode characters, pretend it's an A - curChar = 'A'; - - mJumpToLetterList = std::make_shared(mWindow, "JUMP TO LETTER", false); - for(char c = 'A'; c <= 'Z'; c++) - mJumpToLetterList->add(std::string(1, c), c, c == curChar); - ComponentListRow row; - row.addElement(std::make_shared(mWindow, "JUMP TO LETTER", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); - row.addElement(mJumpToLetterList, false); - row.input_handler = [&](InputConfig* config, Input input) { - if(config->isMappedTo("a", input) && input.value) - { - jumpToLetter(); - return true; - } - else if(mJumpToLetterList->input(config, input)) - { - return true; - } - return false; - }; - mMenu.addRow(row); - + // volume + mVolume = std::make_shared(mWindow, 0.f, 100.f, 1.f, "%"); + mVolume->setValue((float)VolumeControl::getInstance()->getVolume()); + mMenu.addWithLabel("SYSTEM VOLUME", mVolume); // sort list by mListSort = std::make_shared(mWindow, "SORT GAMES BY", false); @@ -48,42 +41,228 @@ GuiGamelistOptions::GuiGamelistOptions(Window* window, SystemData* system) : Gui } mMenu.addWithLabel("SORT GAMES BY", mListSort); + mFoldersFirst = std::make_shared(mWindow); + mFoldersFirst->setState(Settings::getInstance()->getBool("SortFoldersFirst")); + mMenu.addWithLabel("SORT FOLDERS FIRST", mFoldersFirst); + if (system) + { - // edit game metadata - row.elements.clear(); - row.addElement(std::make_shared(mWindow, "EDIT THIS GAME'S METADATA", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); - row.addElement(makeArrow(mWindow), false); - row.makeAcceptInputHandler(std::bind(&GuiGamelistOptions::openMetaDataEd, this)); - mMenu.addRow(row); +/* + // jump to letter + char curChar = toupper(getGamelist()->getCursor().getName()[0]); + if(curChar < 'A' || curChar > 'Z') // in the case of unicode characters, pretend it's an A + curChar = 'A'; + + mJumpToLetterList = std::make_shared(mWindow, "JUMP TO LETTER", false); + for(char c = 'A'; c <= 'Z'; c++) + mJumpToLetterList->add(std::string(1, c), c, c == curChar); + + row.addElement(std::make_shared(mWindow, "JUMP TO LETTER", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + row.addElement(mJumpToLetterList, false); + row.input_handler = [&](InputConfig* config, Input input) { + if(config->isMappedTo("a", input) && input.value) + { + jumpToLetter(); + return true; + } + else if(mJumpToLetterList->input(config, input)) + { + return true; + } + return false; + }; + mMenu.addRow(row); +*/ + + + addEntry("EDIT", 0x777777FF, true, + [this] { + auto s = new GuiSettings(mWindow, "EDIT"); + ComponentListRow row; + // edit game metadata + if(getGamelist()->validCursor()) + { + const FileData& file = getGamelist()->getCursor(); + row.elements.clear(); + row.addElement(std::make_shared(mWindow, "EDIT THIS ENTRY'S METADATA", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + row.addElement(makeArrow(mWindow), false); + row.makeAcceptInputHandler(std::bind(&GuiGamelistOptions::openMetaDataEd, this)); + s->addRow(row); + + + row.elements.clear(); + auto addTagFunc= [this, &file](const std::string& tag) { + SystemManager::getInstance()->database().setFileTag(file.getFileID(),file.getSystemID(),tag,true); + }; + auto removeTagFunc= [this, &file](const std::string& tag) { + SystemManager::getInstance()->database().setFileTag(file.getFileID(),file.getSystemID(),tag,false); + }; + + row.makeAcceptInputHandler([this, &file, addTagFunc] { + mWindow->pushGui(new GuiTextEditPopup(mWindow, "ENTER NEW TAG", "", addTagFunc, false)); + }); + auto addTag = std::make_shared(mWindow, "ADD TAG",Font::get(FONT_SIZE_MEDIUM), 0x777777FF); + auto bracket = makeArrow(mWindow); + row.addElement(addTag, true); + row.addElement(bracket, false); + s->addRow(row); + row.elements.clear(); + row.makeAcceptInputHandler([this, &file, removeTagFunc] { + mWindow->pushGui(new GuiTextEditPopup(mWindow, "ENTER TAG TO REMOVE", "", removeTagFunc, false)); + }); + auto removeTag = std::make_shared(mWindow, "REMOVE TAG",Font::get(FONT_SIZE_MEDIUM), 0x777777FF); + bracket = makeArrow(mWindow); + row.addElement(removeTag, true); + row.addElement(bracket, false); + s->addRow(row); + } + // add a filter entry to our current list + row.elements.clear(); + row.addElement(std::make_shared(mWindow, "ADD FILTER", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + row.addElement(makeArrow(mWindow), false); + row.makeAcceptInputHandler(std::bind(&GuiGamelistOptions::openFilterAddStart, this)); + s->addRow(row); + mWindow->pushGui(s); + }); + } //End Gamelist Specific entries + + addEntry("SETTINGS", 0x777777FF, true, + [this] { + auto s = new GuiMenu(mWindow); + mWindow->pushGui(s); + }); + addEntry("QUIT", 0x777777FF, true, + [this] { + auto s = new GuiSettings(mWindow, "QUIT"); + + Window* window = mWindow; + + ComponentListRow row; + row.makeAcceptInputHandler([window] { + window->pushGui(new GuiMsgBox(window, "REALLY RESTART?", "YES", + [] { + if(runRestartCommand() != 0) + LOG(LogWarning) << "Restart terminated with non-zero result!"; + }, "NO", nullptr)); + }); + row.addElement(std::make_shared(window, "RESTART SYSTEM", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + s->addRow(row); + + row.elements.clear(); + row.makeAcceptInputHandler([window] { + window->pushGui(new GuiMsgBox(window, "REALLY SHUTDOWN?", "YES", + [] { + if(runShutdownCommand() != 0) + LOG(LogWarning) << "Shutdown terminated with non-zero result!"; + }, "NO", nullptr)); + }); + row.addElement(std::make_shared(window, "SHUTDOWN SYSTEM", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + s->addRow(row); + + if(Settings::getInstance()->getBool("ShowExit")) + { + row.elements.clear(); + row.makeAcceptInputHandler([window] { + window->pushGui(new GuiMsgBox(window, "REALLY QUIT?", "YES", + [] { + SDL_Event ev; + ev.type = SDL_QUIT; + SDL_PushEvent(&ev); + }, "NO", nullptr)); + }); + row.addElement(std::make_shared(window, "QUIT EMULATIONSTATION", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); + s->addRow(row); + } + + mWindow->pushGui(s); + }); + mMenu.addButton("BACK", "go back", [this] { delete this; }); // center the menu - setSize((float)Renderer::getScreenWidth(), (float)Renderer::getScreenHeight()); - mMenu.setPosition((mSize.x() - mMenu.getSize().x()) / 2, (mSize.y() - mMenu.getSize().y()) / 2); + setSize(mMenu.getSize()); + mMenu.setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, Renderer::getScreenHeight() * 0.15f); } GuiGamelistOptions::~GuiGamelistOptions() { // apply sort if it changed - if(mListSort->getSelected() != Settings::getInstance()->getInt("SortTypeIndex")) + bool dirty = mListSort->getSelected() != Settings::getInstance()->getInt("SortTypeIndex"); + dirty = dirty || (mFoldersFirst->getState() != Settings::getInstance()->getBool("SortFoldersFirst")); + if(dirty) { Settings::getInstance()->setInt("SortTypeIndex", mListSort->getSelected()); + Settings::getInstance()->setBool("SortFoldersFirst", mFoldersFirst->getState()); ViewController::get()->onFilesChanged(NULL); } + VolumeControl::getInstance()->setVolume((int)round(mVolume->getValue())); +} + + +void GuiGamelistOptions::addEntry(const char* name, unsigned int color, bool add_arrow, const std::function& func) +{ + std::shared_ptr font = Font::get(FONT_SIZE_MEDIUM); + + // populate the list + ComponentListRow row; + row.addElement(std::make_shared(mWindow, name, font, color), true); + + if(add_arrow) + { + std::shared_ptr bracket = makeArrow(mWindow); + row.addElement(bracket, false); + } + + row.makeAcceptInputHandler(func); + + mMenu.addRow(row); } void GuiGamelistOptions::openMetaDataEd() { // open metadata editor + if(!getGamelist()->validCursor()) return; const FileData& file = getGamelist()->getCursor(); ScraperSearchParams p(file.getSystem(), file); + auto deleteFunc = [this, file] { + boost::filesystem::remove(file.getPath()); // actually delete the file on the filesystem + SystemManager::getInstance()->database().removeEntry(file); // update the database + getGamelist()->onFilesChanged(); // tell the view + }; + auto removeFunc = [this, file] { + SystemManager::getInstance()->database().removeEntry(file); // update the database + getGamelist()->onFilesChanged(); // tell the view + }; mWindow->pushGui(new GuiMetaDataEd(mWindow, file, - std::bind(&IGameListView::onMetaDataChanged, getGamelist(), file), [this, file] { - boost::filesystem::remove(file.getPath()); // actually delete the file on the filesystem - SystemManager::getInstance()->database().updateExists(file); // update the database - getGamelist()->onFilesChanged(); // tell the view - })); + std::bind(&IGameListView::onMetaDataChanged, getGamelist(), file), deleteFunc, removeFunc)); +} + +void GuiGamelistOptions::openFilterAddStart() +{ + // open a dialog to add a filter + const FileData& file = getGamelist()->getParentCursor(); + auto updateVal = [this, file](const std::string& filterid) { + this->openFilterAdd(file,filterid); + }; + mWindow->pushGui(new GuiTextEditPopup(mWindow, "ENTER FILTER'S FILE NAME", "", updateVal, false)); + + + } +void GuiGamelistOptions::openFilterAdd(const FileData& fileParent, const std::string &filterid) +{ + if(filterid.empty()) return; + FileData filter(fileParent.getFileID() + "/" + filterid, fileParent.getSystemID(), FILTER); + MetaDataMap emptyfiltermetadata(FILTER_METADATA); + emptyfiltermetadata.set("name",filterid); + filter.set_metadata(emptyfiltermetadata); + auto deleteFunc = [this, filter] { + SystemManager::getInstance()->database().removeEntry(filter); // update the database + getGamelist()->onFilesChanged(); // tell the view + }; + mWindow->pushGui(new GuiMetaDataEd(mWindow, filter, + std::bind(&IGameListView::onMetaDataChanged, getGamelist(), filter), deleteFunc, nullptr)); +} void GuiGamelistOptions::jumpToLetter() { char letter = mJumpToLetterList->getSelected(); @@ -125,11 +304,21 @@ void GuiGamelistOptions::jumpToLetter() bool GuiGamelistOptions::input(InputConfig* config, Input input) { - if((config->isMappedTo("b", input) || config->isMappedTo("select", input)) && input.value) + if((config->isMappedTo("b", input)) && input.value) { delete this; return true; } + + if((config->isMappedTo("select", input) || config->isMappedTo("start",input)) && input.value != 0) + { + // close everything + Window* window = mWindow; + while(window->peekGui() && window->peekGui() != ViewController::get()) + delete window->peekGui(); + return true; + } + return mMenu.input(config, input); } @@ -137,7 +326,8 @@ bool GuiGamelistOptions::input(InputConfig* config, Input input) std::vector GuiGamelistOptions::getHelpPrompts() { auto prompts = mMenu.getHelpPrompts(); - prompts.push_back(HelpPrompt("b", "close")); + prompts.push_back(HelpPrompt("b", "back")); + prompts.push_back(HelpPrompt("select", "close")); return prompts; } diff --git a/es-app/src/guis/GuiGamelistOptions.h b/es-app/src/guis/GuiGamelistOptions.h index 51949caf60..6b25965002 100644 --- a/es-app/src/guis/GuiGamelistOptions.h +++ b/es-app/src/guis/GuiGamelistOptions.h @@ -1,6 +1,9 @@ +#pragma once #include "GuiComponent.h" #include "components/MenuComponent.h" #include "components/OptionListComponent.h" +#include "components/SliderComponent.h" +#include "components/SwitchComponent.h" #include "GamelistDB.h" class IGameListView; @@ -15,7 +18,10 @@ class GuiGamelistOptions : public GuiComponent virtual std::vector getHelpPrompts() override; private: + void addEntry(const char* name, unsigned int color, bool add_arrow, const std::function& func); void openMetaDataEd(); + void openFilterAddStart(); + void openFilterAdd(const FileData& file, const std::string &filterid); void jumpToLetter(); MenuComponent mMenu; @@ -25,6 +31,9 @@ class GuiGamelistOptions : public GuiComponent typedef OptionListComponent SortList; std::shared_ptr mListSort; + + std::shared_ptr mVolume; + std::shared_ptr mFoldersFirst; SystemData* mSystem; IGameListView* getGamelist(); diff --git a/es-app/src/guis/GuiMenu.cpp b/es-app/src/guis/GuiMenu.cpp index ff104f2ad2..da918b4d5f 100644 --- a/es-app/src/guis/GuiMenu.cpp +++ b/es-app/src/guis/GuiMenu.cpp @@ -6,6 +6,7 @@ #include "Settings.h" #include "guis/GuiMsgBox.h" #include "guis/GuiSettings.h" +#include "guis/GuiRefreshDatabase.h" #include "guis/GuiScraperStart.h" #include "guis/GuiDetectDevice.h" #include "views/ViewController.h" @@ -21,19 +22,25 @@ #include "scrapers/GamesDBScraper.h" #include "scrapers/TheArchiveScraper.h" -GuiMenu::GuiMenu(Window* window) : GuiComponent(window), mMenu(window, "MAIN MENU"), mVersion(window) +GuiMenu::GuiMenu(Window* window) : GuiComponent(window), mMenu(window, "SETTINGS"), mVersion(window) { - // MAIN MENU + // SETTINGS MENU - // SCRAPER > - // SOUND SETTINGS > - // UI SETTINGS > - // CONFIGURE INPUT > - // QUIT > + // SCRAPER + // SOUND + // UI + // CONFIGURE INPUT // [version] - auto openScrapeNow = [this] { mWindow->pushGui(new GuiScraperStart(mWindow)); }; + + + auto openScrapeNow = [this] { + mWindow->pushGui(new GuiScraperStart(mWindow)); + }; + auto openRefreshDatabase = [this] { + mWindow->pushGui(new GuiRefreshDatabase(mWindow)); + }; auto importXMLAndClose = [this] { SystemManager::getInstance()->importGamelistXML(false); @@ -42,45 +49,68 @@ GuiMenu::GuiMenu(Window* window) : GuiComponent(window), mMenu(window, "MAIN MEN while(windowCopy->peekGui() != ViewController::get()) delete windowCopy->peekGui(); }; - addEntry("SCRAPER", 0x777777FF, true, - [this, openScrapeNow, importXMLAndClose] { - auto s = new GuiSettings(mWindow, "SCRAPER"); - - // scrape from - auto scraper_list = std::make_shared< OptionListComponent< std::string > >(mWindow, "SCRAPE FROM", false); - std::vector scrapers = getScraperList(); - for(auto it = scrapers.begin(); it != scrapers.end(); it++) - scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper")); - - s->addWithLabel("SCRAPE FROM", scraper_list); - s->addSaveFunc([scraper_list] { Settings::getInstance()->setString("Scraper", scraper_list->getSelected()); }); - - // scrape ratings - auto scrape_ratings = std::make_shared(mWindow); - scrape_ratings->setState(Settings::getInstance()->getBool("ScrapeRatings")); - s->addWithLabel("SCRAPE RATINGS", scrape_ratings); - s->addSaveFunc([scrape_ratings] { Settings::getInstance()->setBool("ScrapeRatings", scrape_ratings->getState()); }); - - // scrape now + + + addEntry("DATABASE", 0x777777FF, true, + [this, openRefreshDatabase, importXMLAndClose, openScrapeNow] { + auto s = new GuiSettings(mWindow, "UPDATE DATABASE"); + + // scan filesystem ComponentListRow row; - std::function openAndSave = openScrapeNow; - openAndSave = [s, openAndSave] { s->save(); openAndSave(); }; - row.makeAcceptInputHandler(openAndSave); + row.makeAcceptInputHandler(openRefreshDatabase); - auto scrape_now = std::make_shared(mWindow, "SCRAPE NOW", Font::get(FONT_SIZE_MEDIUM), 0x777777FF); + auto scan_files = std::make_shared(mWindow, "SCAN FILE SYSTEM", Font::get(FONT_SIZE_MEDIUM), 0x777777FF); auto bracket = makeArrow(mWindow); - row.addElement(scrape_now, true); + row.addElement(scan_files, true); + row.addElement(bracket, false); + s->addRow(row); + + // open scraper + row.elements.clear(); + row.makeAcceptInputHandler([this, openScrapeNow] { + auto s = new GuiSettings(mWindow, "SCRAPER"); + + // scrape from + auto scraper_list = std::make_shared< OptionListComponent< std::string > >(mWindow, "SCRAPE FROM", false); + std::vector scrapers = getScraperList(); + for(auto it = scrapers.begin(); it != scrapers.end(); it++) + scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper")); + + s->addWithLabel("SCRAPE FROM", scraper_list); + s->addSaveFunc([scraper_list] { Settings::getInstance()->setString("Scraper", scraper_list->getSelected()); }); + + // scrape ratings + auto scrape_ratings = std::make_shared(mWindow); + scrape_ratings->setState(Settings::getInstance()->getBool("ScrapeRatings")); + s->addWithLabel("SCRAPE RATINGS", scrape_ratings); + s->addSaveFunc([scrape_ratings] { Settings::getInstance()->setBool("ScrapeRatings", scrape_ratings->getState()); }); + + // scrape now + ComponentListRow row; + std::function openAndSave = openScrapeNow; + openAndSave = [s, openAndSave] { s->save(); openAndSave(); }; + row.makeAcceptInputHandler(openAndSave); + + auto scrape_now = std::make_shared(mWindow, "SCRAPE NOW", Font::get(FONT_SIZE_MEDIUM), 0x777777FF); + auto bracket = makeArrow(mWindow); + row.addElement(scrape_now, true); + row.addElement(bracket, false); + s->addRow(row); + + mWindow->pushGui(s); + }); + auto run_scanner = std::make_shared(mWindow, "SCRAPER",Font::get(FONT_SIZE_MEDIUM), 0x777777FF); + bracket = makeArrow(mWindow); + row.addElement(run_scanner, true); row.addElement(bracket, false); s->addRow(row); // import XML row.elements.clear(); - row.makeAcceptInputHandler(importXMLAndClose); + row.makeAcceptInputHandler(importXMLAndClose); auto import_xml = std::make_shared(mWindow, "IMPORT FROM XML", Font::get(FONT_SIZE_MEDIUM), 0x777777FF); - bracket = makeArrow(mWindow); row.addElement(import_xml, true); - row.addElement(bracket, false); s->addRow(row); mWindow->pushGui(s); @@ -90,12 +120,6 @@ GuiMenu::GuiMenu(Window* window) : GuiComponent(window), mMenu(window, "MAIN MEN [this] { auto s = new GuiSettings(mWindow, "SOUND SETTINGS"); - // volume - auto volume = std::make_shared(mWindow, 0.f, 100.f, 1.f, "%"); - volume->setValue((float)VolumeControl::getInstance()->getVolume()); - s->addWithLabel("SYSTEM VOLUME", volume); - s->addSaveFunc([volume] { VolumeControl::getInstance()->setVolume((int)round(volume->getValue())); }); - // disable sounds auto sounds_enabled = std::make_shared(mWindow); sounds_enabled->setState(Settings::getInstance()->getBool("EnableSounds")); @@ -189,56 +213,14 @@ GuiMenu::GuiMenu(Window* window) : GuiComponent(window), mMenu(window, "MAIN MEN mWindow->pushGui(new GuiDetectDevice(mWindow, false, nullptr)); }); - addEntry("QUIT", 0x777777FF, true, - [this] { - auto s = new GuiSettings(mWindow, "QUIT"); - - Window* window = mWindow; - ComponentListRow row; - row.makeAcceptInputHandler([window] { - window->pushGui(new GuiMsgBox(window, "REALLY RESTART?", "YES", - [] { - if(runRestartCommand() != 0) - LOG(LogWarning) << "Restart terminated with non-zero result!"; - }, "NO", nullptr)); - }); - row.addElement(std::make_shared(window, "RESTART SYSTEM", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); - s->addRow(row); - row.elements.clear(); - row.makeAcceptInputHandler([window] { - window->pushGui(new GuiMsgBox(window, "REALLY SHUTDOWN?", "YES", - [] { - if(runShutdownCommand() != 0) - LOG(LogWarning) << "Shutdown terminated with non-zero result!"; - }, "NO", nullptr)); - }); - row.addElement(std::make_shared(window, "SHUTDOWN SYSTEM", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); - s->addRow(row); - - if(Settings::getInstance()->getBool("ShowExit")) - { - row.elements.clear(); - row.makeAcceptInputHandler([window] { - window->pushGui(new GuiMsgBox(window, "REALLY QUIT?", "YES", - [] { - SDL_Event ev; - ev.type = SDL_QUIT; - SDL_PushEvent(&ev); - }, "NO", nullptr)); - }); - row.addElement(std::make_shared(window, "QUIT EMULATIONSTATION", Font::get(FONT_SIZE_MEDIUM), 0x777777FF), true); - s->addRow(row); - } - - mWindow->pushGui(s); - }); + mMenu.addButton("BACK", "go back", [this] { delete this; }); mVersion.setFont(Font::get(FONT_SIZE_SMALL)); mVersion.setColor(0xC6C6C6FF); - mVersion.setText("EMULATIONSTATION V" + strToUpper(PROGRAM_VERSION_STRING)); - mVersion.setAlignment(ALIGN_CENTER); + mVersion.setText("EMULATIONSTATION V" + strToUpper(PROGRAM_VERSION_STRING) + " "); + mVersion.setAlignment(ALIGN_RIGHT); addChild(&mMenu); addChild(&mVersion); @@ -277,11 +259,20 @@ bool GuiMenu::input(InputConfig* config, Input input) if(GuiComponent::input(config, input)) return true; - if((config->isMappedTo("b", input) || config->isMappedTo("start", input)) && input.value != 0) + if((config->isMappedTo("b", input)) && input.value) { delete this; return true; } + + if((config->isMappedTo("select", input) || config->isMappedTo("start",input)) && input.value != 0) + { + // close everything + Window* window = mWindow; + while(window->peekGui() && window->peekGui() != ViewController::get()) + delete window->peekGui(); + return true; + } return false; } @@ -291,6 +282,6 @@ std::vector GuiMenu::getHelpPrompts() std::vector prompts; prompts.push_back(HelpPrompt("up/down", "choose")); prompts.push_back(HelpPrompt("a", "select")); - prompts.push_back(HelpPrompt("start", "close")); + prompts.push_back(HelpPrompt("select", "close")); return prompts; } diff --git a/es-app/src/guis/GuiMetaDataEd.cpp b/es-app/src/guis/GuiMetaDataEd.cpp index 7c97bb57cc..426c3754ac 100644 --- a/es-app/src/guis/GuiMetaDataEd.cpp +++ b/es-app/src/guis/GuiMetaDataEd.cpp @@ -12,11 +12,13 @@ #include "components/DateTimeComponent.h" #include "components/RatingComponent.h" #include "guis/GuiTextEditPopup.h" +#include "SystemManager.h" + using namespace Eigen; GuiMetaDataEd::GuiMetaDataEd(Window* window, const FileData& file, - const std::function& saveCallback, const std::function& deleteFunc) + const std::function& saveCallback, const std::function& deleteFunc, const std::function& removeFunc) : GuiComponent(window), mFile(file), mMetaData(mFile.get_metadata()), @@ -26,7 +28,7 @@ GuiMetaDataEd::GuiMetaDataEd(Window* window, const FileData& file, mBackground(window, ":/frame.png"), mGrid(window, Vector2i(1, 3)), - mSavedCallback(saveCallback), mDeleteFunc(deleteFunc) + mSavedCallback(saveCallback), mDeleteFunc(deleteFunc), mRemoveFunc(removeFunc) { addChild(&mBackground); addChild(&mGrid); @@ -83,7 +85,7 @@ GuiMetaDataEd::GuiMetaDataEd(Window* window, const FileData& file, row.addElement(ed, false); auto spacer = std::make_shared(mWindow); - spacer->setSize(Renderer::getScreenWidth() * 0.0025f, 0); + spacer->setSize(Renderer::getScreenWidth() * 0.025f, 0); row.addElement(spacer, false); // pass input to the actual DateTimeComponent instead of the spacer @@ -137,6 +139,12 @@ GuiMetaDataEd::GuiMetaDataEd(Window* window, const FileData& file, buttons.push_back(std::make_shared(mWindow, "SAVE", "save", [&] { save(); delete this; })); buttons.push_back(std::make_shared(mWindow, "CANCEL", "cancel", [&] { delete this; })); + if (mRemoveFunc) + { + auto removeAndClose = [&] { mRemoveFunc(); delete this; }; + auto removeBtnFunc = [this, removeAndClose] { mWindow->pushGui(new GuiMsgBox(mWindow, "THIS WILL REMOVE THIS ENTRY FROM THE DATABASE, BUT WILL LEAVE THE FILE ON THE FILE SYSTEM.\nARE YOU SURE?", "YES", removeAndClose, "NO", nullptr)); }; + buttons.push_back(std::make_shared(mWindow, "REMOVE", "remove", removeBtnFunc)); + } if(mDeleteFunc) { auto deleteFileAndSelf = [&] { mDeleteFunc(); delete this; }; diff --git a/es-app/src/guis/GuiMetaDataEd.h b/es-app/src/guis/GuiMetaDataEd.h index 11056d3f73..77ae39fbb0 100644 --- a/es-app/src/guis/GuiMetaDataEd.h +++ b/es-app/src/guis/GuiMetaDataEd.h @@ -11,7 +11,7 @@ class GuiMetaDataEd : public GuiComponent { public: GuiMetaDataEd(Window* window, const FileData& file, - const std::function& savedCallback, const std::function& deleteFunc); + const std::function& savedCallback, const std::function& deleteFunc, const std::function& removeFunc); bool input(InputConfig* config, Input input) override; void onSizeChanged() override; @@ -42,4 +42,5 @@ class GuiMetaDataEd : public GuiComponent std::function mSavedCallback; std::function mDeleteFunc; + std::function mRemoveFunc; }; diff --git a/es-app/src/guis/GuiRefreshDatabase.cpp b/es-app/src/guis/GuiRefreshDatabase.cpp new file mode 100644 index 0000000000..2290f1e7bc --- /dev/null +++ b/es-app/src/guis/GuiRefreshDatabase.cpp @@ -0,0 +1,121 @@ +#include "guis/GuiRefreshDatabase.h" +#include "guis/GuiMsgBox.h" +#include "views/ViewController.h" +#include "SystemManager.h" + +#include "components/TextComponent.h" +#include "components/OptionListComponent.h" +#include "components/SwitchComponent.h" + +GuiRefreshDatabase::GuiRefreshDatabase(Window* window) : GuiComponent(window), + mMenu(window, "REFRESH DATA BASE") +{ + addChild(&mMenu); + + + //add systems (If we are on a system, do it alone. Otherwise, do all) + mSystems = std::make_shared< OptionListComponent >(mWindow, "SCAN THESE SYSTEMS", true); + SystemData* system = ViewController::get()->getSystem(); + for(auto it = SystemManager::getInstance()->getSystems().begin(); it != SystemManager::getInstance()->getSystems().end(); it++) + { + if(!(*it)->hasPlatformId(PlatformIds::PLATFORM_IGNORE)) + mSystems->add((*it)->getFullName(), *it, !system || (*it == system)); + } + mMenu.addWithLabel("Systems", mSystems); + + //Scan for new files to add to the database. + mAddFiles = std::make_shared(mWindow); + mAddFiles->setState(true); + mMenu.addWithLabel("Look for new files", mAddFiles); + + //Mostly useful when Remove Nonexisting is enabled + mCheckExists = std::make_shared(mWindow); + mCheckExists->setState(false); + mMenu.addWithLabel("Check whether files exist", mCheckExists); + + //Potentially deletes data, so default to false. + mRemoveNonexisting = std::make_shared(mWindow); + mRemoveNonexisting->setState(false); + mMenu.addWithLabel("Remove non-existing files", mRemoveNonexisting); + + mMenu.addButton("START", "start", std::bind(&GuiRefreshDatabase::pressedStart, this)); + mMenu.addButton("BACK", "back", [&] { delete this; }); + + mMenu.setPosition((Renderer::getScreenWidth() - mMenu.getSize().x()) / 2, Renderer::getScreenHeight() * 0.15f); +} + +void GuiRefreshDatabase::pressedStart() +{ + if(!mCheckExists->getState() && mRemoveNonexisting->getState() ) + { + mWindow->pushGui(new GuiMsgBox(mWindow, + strToUpper("Warning: The remove action will not be performed since the check is not being performed.\nContinue anyway?"), + "YES", std::bind(&GuiRefreshDatabase::start, this), + "NO", nullptr)); + return; + } + + start(); +} + +void GuiRefreshDatabase::start() +{ + bool addFiles = mAddFiles->getState(); + bool checkExists = mCheckExists->getState(); + bool removeNonexisting = checkExists && mRemoveNonexisting->getState(); + std::vector systems = mSystems->getSelectedObjects(); + + if((!addFiles && !checkExists && !removeNonexisting) || systems.empty()) + { + mWindow->pushGui(new GuiMsgBox(mWindow, + "NOTHING TO BE DONE.\nSELECT AT LEAST ONE ACTION AND AT LEAST ONE SYSTEM.")); + }else{ + auto databaseptr = &SystemManager::getInstance()->database(); + int start = databaseptr->totalChanges(); + for(auto sys = systems.begin(); sys != systems.end(); sys++) + { + if(addFiles) databaseptr->addMissingFiles(*sys); + if(checkExists) databaseptr->updateExists(*sys); + if(removeNonexisting) databaseptr->removeNonexisting(*sys); + ViewController::get()->getGameListView(*sys).get()->onFilesChanged(); + } + int updates = databaseptr->totalChanges() - start; + mWindow->pushGui(new GuiMsgBox(mWindow, + "NUMBER OF DATABASE CHANGES: " + std::to_string(updates))); + delete this; + } +} + + + +bool GuiRefreshDatabase::input(InputConfig* config, Input input) +{ + bool consumed = GuiComponent::input(config, input); + if(consumed) + return true; + + if(input.value != 0 && config->isMappedTo("b", input)) + { + delete this; + return true; + } + + if((config->isMappedTo("start", input) || config->isMappedTo("select",input)) && input.value != 0) + { + // close everything + Window* window = mWindow; + while(window->peekGui() && window->peekGui() != ViewController::get()) + delete window->peekGui(); + } + + + return false; +} + +std::vector GuiRefreshDatabase::getHelpPrompts() +{ + std::vector prompts = mMenu.getHelpPrompts(); + prompts.push_back(HelpPrompt("b", "back")); + prompts.push_back(HelpPrompt("select", "close")); + return prompts; +} diff --git a/es-app/src/guis/GuiRefreshDatabase.h b/es-app/src/guis/GuiRefreshDatabase.h new file mode 100644 index 0000000000..b205766a67 --- /dev/null +++ b/es-app/src/guis/GuiRefreshDatabase.h @@ -0,0 +1,35 @@ +#pragma once + +#include "GuiComponent.h" +#include "SystemData.h" +#include "components/MenuComponent.h" +#include + + +template +class OptionListComponent; + +class SwitchComponent; + +//Allows user to run the expensive file systems scans manually. +//Takes the current system to select only it by default. If nullptr, all are selected. +class GuiRefreshDatabase : public GuiComponent +{ +public: + GuiRefreshDatabase(Window* window); + + bool input(InputConfig* config, Input input) override; + + virtual std::vector getHelpPrompts() override; + +private: + void pressedStart(); + void start(); + + std::shared_ptr< OptionListComponent > mSystems; + std::shared_ptr mAddFiles; + std::shared_ptr mCheckExists; + std::shared_ptr mRemoveNonexisting; + + MenuComponent mMenu; +}; diff --git a/es-app/src/guis/GuiScraperStart.cpp b/es-app/src/guis/GuiScraperStart.cpp index 0533353a7d..fb3f7deb97 100644 --- a/es-app/src/guis/GuiScraperStart.cpp +++ b/es-app/src/guis/GuiScraperStart.cpp @@ -21,12 +21,15 @@ GuiScraperStart::GuiScraperStart(Window* window) : GuiComponent(window), [](SystemData*, const FileData& g) -> bool { return g.get_metadata().get("image").empty(); }, true); mMenu.addWithLabel("Filter", mFilters); - //add systems (all with a platformid specified selected) + //add systems (only current system, or all with a platformid specified selected) + SystemData* currentSystem = ViewController::get()->getSystem(); mSystems = std::make_shared< OptionListComponent >(mWindow, "SCRAPE THESE SYSTEMS", true); for(auto it = SystemManager::getInstance()->getSystems().begin(); it != SystemManager::getInstance()->getSystems().end(); it++) { + bool selectSystem = ((*it) == currentSystem); + if(!currentSystem) selectSystem = !(*it)->getPlatformIds().empty(); if(!(*it)->hasPlatformId(PlatformIds::PLATFORM_IGNORE)) - mSystems->add((*it)->getFullName(), *it, !(*it)->getPlatformIds().empty()); + mSystems->add((*it)->getFullName(), *it, selectSystem); } mMenu.addWithLabel("Systems", mSystems); @@ -104,7 +107,7 @@ bool GuiScraperStart::input(InputConfig* config, Input input) return true; } - if(config->isMappedTo("start", input) && input.value != 0) + if((config->isMappedTo("start", input) || config->isMappedTo("select",input)) && input.value != 0) { // close everything Window* window = mWindow; @@ -120,6 +123,6 @@ std::vector GuiScraperStart::getHelpPrompts() { std::vector prompts = mMenu.getHelpPrompts(); prompts.push_back(HelpPrompt("b", "back")); - prompts.push_back(HelpPrompt("start", "close")); + prompts.push_back(HelpPrompt("select", "close")); return prompts; } diff --git a/es-app/src/guis/GuiSettings.cpp b/es-app/src/guis/GuiSettings.cpp index 806dae2c98..8b7ef9f33b 100644 --- a/es-app/src/guis/GuiSettings.cpp +++ b/es-app/src/guis/GuiSettings.cpp @@ -37,7 +37,7 @@ bool GuiSettings::input(InputConfig* config, Input input) return true; } - if(config->isMappedTo("start", input) && input.value != 0) + if((config->isMappedTo("select", input) || config->isMappedTo("start",input)) && input.value != 0) { // close everything Window* window = mWindow; @@ -54,7 +54,7 @@ std::vector GuiSettings::getHelpPrompts() std::vector prompts = mMenu.getHelpPrompts(); prompts.push_back(HelpPrompt("b", "back")); - prompts.push_back(HelpPrompt("start", "close")); + prompts.push_back(HelpPrompt("select", "close")); return prompts; } diff --git a/es-app/src/scrapers/GamesDBScraper.cpp b/es-app/src/scrapers/GamesDBScraper.cpp index c3461ebf3e..5ae921b5b3 100644 --- a/es-app/src/scrapers/GamesDBScraper.cpp +++ b/es-app/src/scrapers/GamesDBScraper.cpp @@ -61,7 +61,8 @@ const std::map gamesdb_platformid_map = boost::assign:: (TURBOGRAFX_16, "TurboGrafx 16") (WONDERSWAN, "WonderSwan") (WONDERSWAN_COLOR, "WonderSwan Color") - (ZX_SPECTRUM, "Sinclair ZX Spectrum"); + (ZX_SPECTRUM, "Sinclair ZX Spectrum") + (VIRTUAL_BOY, "Nintendo Virtual Boy"); void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr >& requests, diff --git a/es-app/src/views/SystemView.cpp b/es-app/src/views/SystemView.cpp index 6a16f01d2c..7aafb9ee85 100644 --- a/es-app/src/views/SystemView.cpp +++ b/es-app/src/views/SystemView.cpp @@ -59,8 +59,10 @@ void SystemView::populate() e.data.logoSelected = std::shared_ptr(logoSelected); }else{ // no logo in theme; use text + std::string name = (*it)->getName(); + if(name.empty()) name = "all"; TextComponent* text = new TextComponent(mWindow, - (*it)->getName(), + name, Font::get(FONT_SIZE_LARGE), 0x000000FF, ALIGN_CENTER); @@ -68,7 +70,7 @@ void SystemView::populate() e.data.logo = std::shared_ptr(text); TextComponent* textSelected = new TextComponent(mWindow, - (*it)->getName(), + name, Font::get((int)(FONT_SIZE_LARGE * SELECTED_SCALE)), 0x000000FF, ALIGN_CENTER); diff --git a/es-app/src/views/ViewController.cpp b/es-app/src/views/ViewController.cpp index 8236ff1f1b..0111082837 100644 --- a/es-app/src/views/ViewController.cpp +++ b/es-app/src/views/ViewController.cpp @@ -7,6 +7,7 @@ #include "views/gamelist/DetailedGameListView.h" #include "views/gamelist/GridGameListView.h" #include "guis/GuiMenu.h" +#include "guis/GuiGamelistOptions.h" #include "guis/GuiMsgBox.h" #include "animations/LaunchAnimation.h" #include "animations/MoveCameraAnimation.h" @@ -64,7 +65,7 @@ void ViewController::goToSystemView(SystemData* system) systemList->goToSystem(system, false); mCurrentView = systemList; - + mFadeOpacity = 0; playViewTransition(); } @@ -73,7 +74,7 @@ void ViewController::goToNextGameList() assert(mState.viewing == GAME_LIST); SystemData* system = getState().getSystem(); assert(system); - goToGameList(SystemManager::getInstance()->getNext(system)); + goToGameList(SystemManager::getInstance()->getNext(system), 1); } void ViewController::goToPrevGameList() @@ -81,10 +82,10 @@ void ViewController::goToPrevGameList() assert(mState.viewing == GAME_LIST); SystemData* system = getState().getSystem(); assert(system); - goToGameList(SystemManager::getInstance()->getPrev(system)); + goToGameList(SystemManager::getInstance()->getPrev(system), -1); } -void ViewController::goToGameList(SystemData* system) +void ViewController::goToGameList(SystemData* system, int velocity) { if(mState.viewing == SYSTEM_SELECT) { @@ -95,12 +96,27 @@ void ViewController::goToGameList(SystemData* system) sysList->setPosition(sysId * (float)Renderer::getScreenWidth(), sysList->getPosition().y()); offX = sysList->getPosition().x() - offX; mCamera.translation().x() -= offX; + //Recompute meta system entries, so that the system select can be used to refresh them + if(system->isMetaSystem()) + onFilesChanged(system); } mState.viewing = GAME_LIST; mState.system = system; mCurrentView = getGameListView(system); + //If needed, wrap around before scrolling. + if(velocity < 0 && mCurrentView->getPosition().x() > -mCamera.translation().x()) + { + float sceneWidth = (float)Renderer::getScreenWidth() * mGameListViews.size(); + mCamera *= Eigen::Translation3f(-sceneWidth,0,0); + } + if(velocity > 0 && mCurrentView->getPosition().x() < -mCamera.translation().x()) + { + float sceneWidth = (float)Renderer::getScreenWidth() * mGameListViews.size(); + mCamera *= Eigen::Translation3f(sceneWidth,0,0); + } + mFadeOpacity = 0; playViewTransition(); } @@ -175,8 +191,14 @@ void ViewController::onMetaDataChanged(SystemData* system, const FileData& file) if(it != mGameListViews.end()) it->second->onMetaDataChanged(file); } +void ViewController::onStatisticsChanged(SystemData* system, const FileData& file) +{ + auto it = mGameListViews.find(system); + if(it != mGameListViews.end()) + it->second->onStatisticsChanged(file); +} -void ViewController::launch(FileData& game, Eigen::Vector3f center) +void ViewController::launch(const FileData& game, Eigen::Vector3f center) { if(game.getType() != GAME) { @@ -199,22 +221,22 @@ void ViewController::launch(FileData& game, Eigen::Vector3f center) //mFadeOpacity = lerp(0.0f, 1.0f, t*t*t + 1); mFadeOpacity = lerp(0.0f, 1.0f, t); }; - setAnimation(new LambdaAnimation(fadeFunc, 800), 0, [this, game, fadeFunc] + setAnimation(new LambdaAnimation(fadeFunc, 800), 0, [this, &game, fadeFunc] { game.getSystem()->launchGame(mWindow, game); mLockInput = false; setAnimation(new LambdaAnimation(fadeFunc, 800), 0, nullptr, true); - this->onMetaDataChanged(game.getSystem(), game); + this->onStatisticsChanged(game.getSystem(), game); }); }else{ // move camera to zoom in on center + fade out, launch game, come back in - setAnimation(new LaunchAnimation(mCamera, mFadeOpacity, center, 1500), 0, [this, origCamera, center, game] + setAnimation(new LaunchAnimation(mCamera, mFadeOpacity, center, 1500), 0, [this, origCamera, center, &game] { game.getSystem()->launchGame(mWindow, game); mCamera = origCamera; mLockInput = false; setAnimation(new LaunchAnimation(mCamera, mFadeOpacity, center, 600), 0, nullptr, true); - this->onMetaDataChanged(game.getSystem(), game); + this->onStatisticsChanged(game.getSystem(), game); }); } } @@ -268,10 +290,15 @@ bool ViewController::input(InputConfig* config, Input input) return true; // open menu - if(config->isMappedTo("start", input) && input.value != 0) + if(config->isMappedTo("select", input) && input.value != 0) { // open menu - mWindow->pushGui(new GuiMenu(mWindow)); + if(mState.viewing == GAME_LIST) + { + mWindow->pushGui(new GuiGamelistOptions(mWindow, mState.getSystem())); + } else { + mWindow->pushGui(new GuiGamelistOptions(mWindow, nullptr)); + } return true; } @@ -301,7 +328,31 @@ void ViewController::render(const Eigen::Affine3f& parentTrans) // draw systemview getSystemListView()->render(trans); - + Eigen::Affine3f wrapTrans; + Eigen::Vector3f wrapStart; + Eigen::Vector3f wrapEnd; + bool wrapped = false; + + float sceneWidth = (float)Renderer::getScreenWidth() * mGameListViews.size(); + if(viewStart.x() < 0) + { + wrapped = true; + wrapTrans = trans * Eigen::Translation3f(-sceneWidth,0,0); + } + if(viewEnd.x() > sceneWidth) + { + wrapped = true; + wrapTrans = trans * Eigen::Translation3f(sceneWidth,0,0); + } + + if(wrapped) + { + wrapStart = wrapTrans.inverse().translation(); + wrapEnd = wrapTrans.inverse() * Eigen::Vector3f((float)Renderer::getScreenWidth(), (float)Renderer::getScreenHeight(), 0); + } + + + // draw gamelists for(auto it = mGameListViews.begin(); it != mGameListViews.end(); it++) { @@ -312,6 +363,9 @@ void ViewController::render(const Eigen::Affine3f& parentTrans) if(guiEnd.x() >= viewStart.x() && guiEnd.y() >= viewStart.y() && guiStart.x() <= viewEnd.x() && guiStart.y() <= viewEnd.y()) it->second->render(trans); + if(wrapped && guiEnd.x() >= wrapStart.x() && guiEnd.y() >= wrapStart.y() && + guiStart.x() <= wrapEnd.x() && guiStart.y() <= wrapEnd.y()) + it->second->render(wrapTrans); } if(mWindow->peekGui() == this) @@ -399,7 +453,7 @@ std::vector ViewController::getHelpPrompts() return prompts; prompts = mCurrentView->getHelpPrompts(); - prompts.push_back(HelpPrompt("start", "menu")); + prompts.push_back(HelpPrompt("select", "menu")); return prompts; } diff --git a/es-app/src/views/ViewController.h b/es-app/src/views/ViewController.h index d2646ecf6e..07ab358f66 100644 --- a/es-app/src/views/ViewController.h +++ b/es-app/src/views/ViewController.h @@ -27,17 +27,18 @@ class ViewController : public GuiComponent // Navigation. void goToNextGameList(); void goToPrevGameList(); - void goToGameList(SystemData* system); + void goToGameList(SystemData* system, int velocity = 0); void goToSystemView(SystemData* system); void goToStart(); // pass NULL for "all systems" void onFilesChanged(SystemData* system); void onMetaDataChanged(SystemData* system, const FileData& file); + void onStatisticsChanged(SystemData* system, const FileData& file); // Plays a nice launch effect and launches the game at the end of it. // Once the game terminates, plays a return effect. - void launch(FileData& game, Eigen::Vector3f centerCameraOn = Eigen::Vector3f(Renderer::getScreenWidth() / 2.0f, Renderer::getScreenHeight() / 2.0f, 0)); + void launch(const FileData& game, Eigen::Vector3f centerCameraOn = Eigen::Vector3f(Renderer::getScreenWidth() / 2.0f, Renderer::getScreenHeight() / 2.0f, 0)); bool input(InputConfig* config, Input input) override; void update(int deltaTime) override; @@ -63,6 +64,10 @@ class ViewController : public GuiComponent }; inline const State& getState() const { return mState; } + SystemData* getSystem() const { + if(mState.viewing != GAME_LIST) return nullptr; + return mState.getSystem(); + } virtual std::vector getHelpPrompts() override; virtual HelpStyle getHelpStyle() override; diff --git a/es-app/src/views/gamelist/BasicGameListView.cpp b/es-app/src/views/gamelist/BasicGameListView.cpp index fd4552e040..a65f3c31e4 100644 --- a/es-app/src/views/gamelist/BasicGameListView.cpp +++ b/es-app/src/views/gamelist/BasicGameListView.cpp @@ -43,7 +43,7 @@ void BasicGameListView::populateList(const std::vector& files) for(auto it = files.begin(); it != files.end(); it++) { - mList.add(it->getName(), *it, (it->getType() == FOLDER)); + mList.add(it->getName(), *it, (it->getType() == FOLDER || it->getType() == FILTER)); } } @@ -52,6 +52,21 @@ const FileData& BasicGameListView::getCursor() return mList.getSelected(); } +const FileData& BasicGameListView::getParentCursor() +{ + return mCursorStack.top(); +} + +const bool BasicGameListView::validCursor() +{ + return mList.size() > 0; +} + +const bool BasicGameListView::validParentCursor() +{ + return mCursorStack.size() > 0; +} + void BasicGameListView::setCursor(const FileData& cursor) { if(!mList.setCursor(cursor)) @@ -60,7 +75,7 @@ void BasicGameListView::setCursor(const FileData& cursor) } } -void BasicGameListView::launch(FileData& game) +void BasicGameListView::launch(const FileData& game) { ViewController::get()->launch(game); } @@ -74,6 +89,6 @@ std::vector BasicGameListView::getHelpPrompts() prompts.push_back(HelpPrompt("up/down", "choose")); prompts.push_back(HelpPrompt("a", "launch")); prompts.push_back(HelpPrompt("b", "back")); - prompts.push_back(HelpPrompt("select", "options")); + prompts.push_back(HelpPrompt("select", "menu")); return prompts; } diff --git a/es-app/src/views/gamelist/BasicGameListView.h b/es-app/src/views/gamelist/BasicGameListView.h index 4e4067787b..c745ccdef2 100644 --- a/es-app/src/views/gamelist/BasicGameListView.h +++ b/es-app/src/views/gamelist/BasicGameListView.h @@ -17,6 +17,9 @@ class BasicGameListView : public ISimpleGameListView virtual void onThemeChanged(const std::shared_ptr& theme); virtual const FileData& getCursor() override; + virtual const FileData& getParentCursor() override; + virtual const bool validCursor() override; + virtual const bool validParentCursor() override; virtual void setCursor(const FileData& file) override; virtual const char* getName() const override { return "basic"; } @@ -25,7 +28,7 @@ class BasicGameListView : public ISimpleGameListView protected: virtual void populateList(const std::vector& files) override; - virtual void launch(FileData& game) override; + virtual void launch(const FileData& game) override; TextListComponent mList; }; diff --git a/es-app/src/views/gamelist/DetailedGameListView.cpp b/es-app/src/views/gamelist/DetailedGameListView.cpp index e482fbf753..3d37309e39 100644 --- a/es-app/src/views/gamelist/DetailedGameListView.cpp +++ b/es-app/src/views/gamelist/DetailedGameListView.cpp @@ -195,6 +195,7 @@ void DetailedGameListView::updateInfoPanel() if(file->getType() == GAME) { + mLastPlayed.setDisplayMode(DateTimeComponent::DISP_RELATIVE_TO_NOW); mRating.setValue(metadata.get("rating")); mReleaseDate.setValue(metadata.get("releasedate")); mDeveloper.setValue(metadata.get("developer")); @@ -203,6 +204,16 @@ void DetailedGameListView::updateInfoPanel() mPlayers.setValue(metadata.get("players")); mLastPlayed.setValue(metadata.get("lastplayed")); mPlayCount.setValue(metadata.get("playcount")); + }else{ + mLastPlayed.setDisplayMode(DateTimeComponent::DISP_DATE_TIME); + mRating.setValue(""); + mReleaseDate.setValue(""); + mDeveloper.setValue(""); + mPublisher.setValue(""); + mGenre.setValue(""); + mPlayers.setValue(""); + mLastPlayed.setValue(""); + mPlayCount.setValue(""); } fadingOut = false; @@ -233,13 +244,14 @@ void DetailedGameListView::updateInfoPanel() } } -void DetailedGameListView::launch(FileData& game) +void DetailedGameListView::launch(const FileData& game) { Eigen::Vector3f target(Renderer::getScreenWidth() / 2.0f, Renderer::getScreenHeight() / 2.0f, 0); if(mImage.hasImage()) target << mImage.getCenter().x(), mImage.getCenter().y(), 0; ViewController::get()->launch(game, target); + updateInfoPanel(); } // we know we won't be recreated as a new type of gamelist, @@ -254,7 +266,16 @@ void DetailedGameListView::onMetaDataChanged(const FileData& file) { ISimpleGameListView::onMetaDataChanged(file); } +void DetailedGameListView::onStatisticsChanged(const FileData& file) +{ + updateInfoPanel(); +} +void DetailedGameListView::populateList(const std::vector& files) +{ + BasicGameListView::populateList(files); + updateInfoPanel(); +} std::vector DetailedGameListView::getMDLabels() { std::vector ret; diff --git a/es-app/src/views/gamelist/DetailedGameListView.h b/es-app/src/views/gamelist/DetailedGameListView.h index 99cea0e101..26b1356e2e 100644 --- a/es-app/src/views/gamelist/DetailedGameListView.h +++ b/es-app/src/views/gamelist/DetailedGameListView.h @@ -14,11 +14,13 @@ class DetailedGameListView : public BasicGameListView virtual void onFilesChanged() override; virtual void onMetaDataChanged(const FileData& file) override; + virtual void onStatisticsChanged(const FileData& file) override; virtual const char* getName() const override { return "detailed"; } protected: - virtual void launch(FileData& game) override; + virtual void launch(const FileData& game) override; + virtual void populateList(const std::vector& files) override; private: void updateInfoPanel(); diff --git a/es-app/src/views/gamelist/IGameListView.cpp b/es-app/src/views/gamelist/IGameListView.cpp index e0eccadf40..0e95c0e1e9 100644 --- a/es-app/src/views/gamelist/IGameListView.cpp +++ b/es-app/src/views/gamelist/IGameListView.cpp @@ -10,15 +10,9 @@ bool IGameListView::input(InputConfig* config, Input input) { - // select to open GuiGamelistOptions - if(config->isMappedTo("select", input) && input.value) - { - Sound::getFromTheme(mTheme, getName(), "menuOpen")->play(); - mWindow->pushGui(new GuiGamelistOptions(mWindow, this->mRoot.getSystem())); - return true; // Ctrl-R to reload a view when debugging - }else if(Settings::getInstance()->getBool("Debug") && config->getDeviceId() == DEVICE_KEYBOARD && + if(Settings::getInstance()->getBool("Debug") && config->getDeviceId() == DEVICE_KEYBOARD && (SDL_GetModState() & (KMOD_LCTRL | KMOD_RCTRL)) && input.id == SDLK_r && input.value != 0) { LOG(LogDebug) << "reloading view"; diff --git a/es-app/src/views/gamelist/IGameListView.h b/es-app/src/views/gamelist/IGameListView.h index 52fa6f3ed3..8cb9115ab9 100644 --- a/es-app/src/views/gamelist/IGameListView.h +++ b/es-app/src/views/gamelist/IGameListView.h @@ -19,6 +19,7 @@ class IGameListView : public GuiComponent virtual void onFilesChanged() = 0; virtual void onMetaDataChanged(const FileData& file) = 0; + virtual void onStatisticsChanged(const FileData& file) = 0; // Called whenever the theme changes. virtual void onThemeChanged(const std::shared_ptr& theme) = 0; @@ -27,6 +28,9 @@ class IGameListView : public GuiComponent inline const std::shared_ptr& getTheme() const { return mTheme; } virtual const FileData& getCursor() = 0; + virtual const bool validCursor() = 0; + virtual const FileData& getParentCursor() = 0; + virtual const bool validParentCursor() = 0; virtual void setCursor(const FileData& file) = 0; virtual bool input(InputConfig* config, Input input) override; @@ -37,4 +41,5 @@ class IGameListView : public GuiComponent protected: FileData mRoot; std::shared_ptr mTheme; + std::stack mCursorStack; }; diff --git a/es-app/src/views/gamelist/ISimpleGameListView.cpp b/es-app/src/views/gamelist/ISimpleGameListView.cpp index bc7142e0b5..575edc4e58 100644 --- a/es-app/src/views/gamelist/ISimpleGameListView.cpp +++ b/es-app/src/views/gamelist/ISimpleGameListView.cpp @@ -48,6 +48,7 @@ void ISimpleGameListView::onThemeChanged(const std::shared_ptr& theme // but this shouldn't happen very often so we'll just always repopulate void ISimpleGameListView::onFilesChanged() { + if(!validCursor()) { populateList(mCursorStack.top().getChildren()); return;}; FileData cursor = getCursor(); FileData parent = mCursorStack.top(); populateList(parent.getChildren()); @@ -58,31 +59,37 @@ void ISimpleGameListView::onMetaDataChanged(const FileData& file) { onFilesChanged(); } +void ISimpleGameListView::onStatisticsChanged(const FileData& file) +{ + return; +} + bool ISimpleGameListView::input(InputConfig* config, Input input) { if(input.value != 0) { - if(config->isMappedTo("a", input)) + if(validCursor() && config->isMappedTo("a", input) || config->isMappedTo("start", input)) { - FileData cursor = getCursor(); + const FileData& cursor = getCursor(); if(cursor.getType() == GAME) { Sound::getFromTheme(getTheme(), getName(), "launch")->play(); launch(cursor); }else{ - // it's a folder - if(cursor.getChildren().size() > 0) - { + // it's a folder or filter + auto children = cursor.getChildren(); + //if(children.size() > 0) + //{ mCursorStack.push(cursor); - populateList(cursor.getChildren()); - } + populateList(children); + //} } - + return true; }else if(config->isMappedTo("b", input)) { - if(mCursorStack.top() != mRoot) + if(mCursorStack.size() > 1) { FileData old_cursor = mCursorStack.top(); mCursorStack.pop(); diff --git a/es-app/src/views/gamelist/ISimpleGameListView.h b/es-app/src/views/gamelist/ISimpleGameListView.h index e56d6e511c..67cdf644c6 100644 --- a/es-app/src/views/gamelist/ISimpleGameListView.h +++ b/es-app/src/views/gamelist/ISimpleGameListView.h @@ -13,6 +13,7 @@ class ISimpleGameListView : public IGameListView virtual void onFilesChanged(); virtual void onMetaDataChanged(const FileData& file); + virtual void onStatisticsChanged(const FileData& file); // Called whenever the theme changes. virtual void onThemeChanged(const std::shared_ptr& theme); @@ -24,13 +25,11 @@ class ISimpleGameListView : public IGameListView protected: virtual void populateList(const std::vector& files) = 0; - virtual void launch(FileData& game) = 0; + virtual void launch(const FileData& game) = 0; TextComponent mHeaderText; ImageComponent mHeaderImage; ImageComponent mBackground; ThemeExtras mThemeExtras; - - std::stack mCursorStack; }; diff --git a/es-core/src/GuiComponent.h b/es-core/src/GuiComponent.h index 0b8734a2f2..abe484d12a 100644 --- a/es-core/src/GuiComponent.h +++ b/es-core/src/GuiComponent.h @@ -13,6 +13,15 @@ class Font; typedef std::pair HelpPrompt; +enum FocusGainDirection +{ + FOCUS_GAIN_DOWN, + FOCUS_GAIN_UP, + FOCUS_GAIN_LEFT, + FOCUS_GAIN_RIGHT, + FOCUS_GAIN_UNKNOWN, +}; + class GuiComponent { public: @@ -77,6 +86,7 @@ class GuiComponent virtual void onFocusGained() {}; virtual void onFocusLost() {}; + virtual void onFocusGained(FocusGainDirection dir) { onFocusGained(); }; // Default implementation just handles and tags as normalized float pairs. // You probably want to keep this behavior for any derived classes as well as add your own. diff --git a/es-core/src/Settings.cpp b/es-core/src/Settings.cpp index 7d971c0740..92380f7a99 100644 --- a/es-core/src/Settings.cpp +++ b/es-core/src/Settings.cpp @@ -70,6 +70,8 @@ void Settings::setDefaults() mIntMap["ScraperResizeHeight"] = 0; mIntMap["SortTypeIndex"] = 0; + mBoolMap["SortFoldersFirst"] = false; + mStringMap["TransitionStyle"] = "fade"; mStringMap["ThemeSet"] = ""; mStringMap["ScreenSaverBehavior"] = "dim"; diff --git a/es-core/src/Util.h b/es-core/src/Util.h index 261a459096..123b05ec40 100644 --- a/es-core/src/Util.h +++ b/es-core/src/Util.h @@ -8,7 +8,7 @@ // the original gamelist.xml used this format - it's something ISO, I forget what #define LEGACY_TIME_STRING_FORMAT "%Y%m%dT%H%M%S%F%q" // the SQLite database uses this format, as do MD_TIME/MD_DATE entries in the MetaDataMap -#define SQLITE_TIME_STRING_FORMAT "%Y-%m-%d %H:%M:%S.%F" +#define SQLITE_TIME_STRING_FORMAT "%Y-%m-%d %H:%M:%S" std::string strToUpper(const char* from); std::string& strToUpper(std::string& str); diff --git a/es-core/src/components/ComponentGrid.cpp b/es-core/src/components/ComponentGrid.cpp index 89dd157cc0..0d2b60f8c9 100644 --- a/es-core/src/components/ComponentGrid.cpp +++ b/es-core/src/components/ComponentGrid.cpp @@ -98,7 +98,7 @@ void ComponentGrid::setEntry(const std::shared_ptr& comp, const Ei { auto origCursor = mCursor; mCursor = pos; - onCursorMoved(origCursor, mCursor); + onCursorMoved(origCursor, mCursor, Eigen::Vector2i(0,0)); } updateCellComponent(mCells.back()); @@ -264,7 +264,7 @@ void ComponentGrid::resetCursor() { Eigen::Vector2i origCursor = mCursor; mCursor = it->pos; - onCursorMoved(origCursor, mCursor); + onCursorMoved(origCursor, mCursor, Eigen::Vector2i(0,0)); break; } } @@ -273,6 +273,7 @@ void ComponentGrid::resetCursor() bool ComponentGrid::moveCursor(Eigen::Vector2i dir) { assert(dir.x() || dir.y()); + assert(!(dir.x() && dir.y())); const Eigen::Vector2i origCursor = mCursor; @@ -283,6 +284,8 @@ bool ComponentGrid::moveCursor(Eigen::Vector2i dir) while(mCursor.x() >= 0 && mCursor.y() >= 0 && mCursor.x() < mGridSize.x() && mCursor.y() < mGridSize.y()) { mCursor = mCursor + dir; + mCursor(0) = (mCursor(0) + mGridSize.x()) % mGridSize.x(); + mCursor(1) = (mCursor(1) + mGridSize.y()) % mGridSize.y(); Eigen::Vector2i curDirPos = mCursor; @@ -294,9 +297,13 @@ bool ComponentGrid::moveCursor(Eigen::Vector2i dir) cursorEntry = getCellAt(mCursor); if(cursorEntry && cursorEntry->canFocus && cursorEntry != currentCursorEntry) { - onCursorMoved(origCursor, mCursor); + onCursorMoved(origCursor, mCursor, dir); return true; } + if (cursorEntry == currentCursorEntry) + { + return false; + } mCursor += searchAxis; } @@ -309,9 +316,13 @@ bool ComponentGrid::moveCursor(Eigen::Vector2i dir) cursorEntry = getCellAt(mCursor); if(cursorEntry && cursorEntry->canFocus && cursorEntry != currentCursorEntry) { - onCursorMoved(origCursor, mCursor); + onCursorMoved(origCursor, mCursor, dir); return true; } + if (cursorEntry == currentCursorEntry) + { + return false; + } mCursor -= searchAxis; } @@ -389,15 +400,24 @@ void ComponentGrid::textInput(const char* text) selectedEntry->component->textInput(text); } -void ComponentGrid::onCursorMoved(Eigen::Vector2i from, Eigen::Vector2i to) +void ComponentGrid::onCursorMoved(Eigen::Vector2i from, Eigen::Vector2i to, Eigen::Vector2i dir) { GridEntry* cell = getCellAt(from); if(cell) cell->component->onFocusLost(); cell = getCellAt(to); + FocusGainDirection gainDir = FOCUS_GAIN_UNKNOWN; + if(!dir.x() && dir.y() > 0) + gainDir = FOCUS_GAIN_DOWN; + if(!dir.x() && dir.y() < 0) + gainDir = FOCUS_GAIN_UP; + if(!dir.y() && dir.x() < 0) + gainDir = FOCUS_GAIN_LEFT; + if(!dir.y() && dir.x() > 0) + gainDir = FOCUS_GAIN_RIGHT; if(cell) - cell->component->onFocusGained(); + cell->component->onFocusGained(gainDir); updateHelpPrompts(); } @@ -410,7 +430,7 @@ void ComponentGrid::setCursorTo(const std::shared_ptr& comp) { Eigen::Vector2i oldCursor = mCursor; mCursor = it->pos; - onCursorMoved(oldCursor, mCursor); + onCursorMoved(oldCursor, mCursor, Eigen::Vector2i(0,0)); return; } } diff --git a/es-core/src/components/ComponentGrid.h b/es-core/src/components/ComponentGrid.h index ee1547c1cd..b6d6cd233d 100644 --- a/es-core/src/components/ComponentGrid.h +++ b/es-core/src/components/ComponentGrid.h @@ -114,6 +114,6 @@ class ComponentGrid : public GuiComponent std::vector mCells; - void onCursorMoved(Eigen::Vector2i from, Eigen::Vector2i to); + void onCursorMoved(Eigen::Vector2i from, Eigen::Vector2i to, Eigen::Vector2i dir); Eigen::Vector2i mCursor; }; diff --git a/es-core/src/components/ComponentList.cpp b/es-core/src/components/ComponentList.cpp index a16bcb0975..229199ea39 100644 --- a/es-core/src/components/ComponentList.cpp +++ b/es-core/src/components/ComponentList.cpp @@ -54,6 +54,21 @@ void ComponentList::onFocusGained() mFocused = true; } +void ComponentList::onFocusGained(FocusGainDirection dir) +{ + if (dir == FOCUS_GAIN_UP) + { + mCursor = size() - 1; + onCursorChanged(CURSOR_STOPPED); + } + if (dir == FOCUS_GAIN_DOWN) + { + mCursor = 0; + onCursorChanged(CURSOR_STOPPED); + } + mFocused = true; +} + bool ComponentList::input(InputConfig* config, Input input) { if(size() == 0) @@ -200,7 +215,6 @@ void ComponentList::render(const Eigen::Affine3f& parentTrans) // need a function that goes roughly 0x777777 -> 0xFFFFFF // and 0xFFFFFF -> 0x777777 // (1 - dst) + 0x77 - const float selectedRowHeight = getRowHeight(mEntries.at(mCursor).data); Renderer::drawRect(0.0f, mSelectorBarOffset, mSize.x(), selectedRowHeight, 0xFFFFFFFF, GL_ONE_MINUS_DST_COLOR, GL_ZERO); diff --git a/es-core/src/components/ComponentList.h b/es-core/src/components/ComponentList.h index 99f06a4ecc..6c4a485ca6 100644 --- a/es-core/src/components/ComponentList.h +++ b/es-core/src/components/ComponentList.h @@ -57,6 +57,7 @@ class ComponentList : public IList void onSizeChanged() override; void onFocusGained() override; + void onFocusGained(FocusGainDirection dir) override; void onFocusLost() override; bool moveCursor(int amt); diff --git a/es-core/src/components/DateTimeComponent.cpp b/es-core/src/components/DateTimeComponent.cpp index ccfb3bd2fd..5479986ded 100644 --- a/es-core/src/components/DateTimeComponent.cpp +++ b/es-core/src/components/DateTimeComponent.cpp @@ -171,7 +171,7 @@ void DateTimeComponent::setValue(const std::string& val) std::string DateTimeComponent::getValue() const { - return boost::posix_time::to_iso_string(mTime); + return ptime_to_string(mTime, SQLITE_TIME_STRING_FORMAT); } DateTimeComponent::DisplayMode DateTimeComponent::getCurrentDisplayMode() const @@ -256,6 +256,8 @@ void DateTimeComponent::updateTextCache() if(mAutoSize) { mSize = mTextCache->metrics.size; + + mAutoSize = false; if(getParent()) getParent()->onSizeChanged(); } @@ -302,6 +304,7 @@ void DateTimeComponent::setFont(std::shared_ptr font) void DateTimeComponent::onSizeChanged() { + mAutoSize = false; updateTextCache(); } diff --git a/es-core/src/components/TextEditComponent.cpp b/es-core/src/components/TextEditComponent.cpp index af57991924..e57cb63a51 100644 --- a/es-core/src/components/TextEditComponent.cpp +++ b/es-core/src/components/TextEditComponent.cpp @@ -26,12 +26,14 @@ TextEditComponent::TextEditComponent(Window* window) : GuiComponent(window), void TextEditComponent::onFocusGained() { mFocused = true; + if(mAutoEdit) startEditing(); mBox.setImagePath(":/textinput_ninepatch_active.png"); } void TextEditComponent::onFocusLost() { mFocused = false; + if(mEditing) stopEditing(); mBox.setImagePath(":/textinput_ninepatch.png"); } @@ -127,10 +129,16 @@ bool TextEditComponent::input(InputConfig* config, Input input) if(config->isMappedTo("up", input)) { - // TODO + if(!isMultiline()) + { + return false; + } }else if(config->isMappedTo("down", input)) { - // TODO + if(!isMultiline()) + { + return false; + } }else if(config->isMappedTo("left", input) || config->isMappedTo("right", input)) { mCursorRepeatDir = config->isMappedTo("left", input) ? -1 : 1; diff --git a/es-core/src/components/TextEditComponent.h b/es-core/src/components/TextEditComponent.h index 56179af1a8..c00f731606 100644 --- a/es-core/src/components/TextEditComponent.h +++ b/es-core/src/components/TextEditComponent.h @@ -25,6 +25,8 @@ class TextEditComponent : public GuiComponent void setValue(const std::string& val) override; std::string getValue() const override; + void setAutoEdit(const bool value) { mAutoEdit = value; }; + inline bool isEditing() const { return mEditing; }; inline const std::shared_ptr& getFont() const { return mFont; } @@ -49,6 +51,7 @@ class TextEditComponent : public GuiComponent std::string mText; bool mFocused; bool mEditing; + bool mAutoEdit = false; //Start editing upon focus int mCursor; // cursor position in characters int mCursorRepeatTimer; diff --git a/es-core/src/guis/GuiTextEditPopup.cpp b/es-core/src/guis/GuiTextEditPopup.cpp index dcb4a47f74..c3a707fbc3 100644 --- a/es-core/src/guis/GuiTextEditPopup.cpp +++ b/es-core/src/guis/GuiTextEditPopup.cpp @@ -14,6 +14,7 @@ GuiTextEditPopup::GuiTextEditPopup(Window* window, const std::string& title, con mText = std::make_shared(mWindow); mText->setValue(initValue); + mText->setAutoEdit(true); if(!multiLine) mText->setCursor(initValue.size()); diff --git a/external/sqlite3/sqlite3.h b/external/sqlite3/sqlite3.h index 023dad1846..7cca0accfa 100644 --- a/external/sqlite3/sqlite3.h +++ b/external/sqlite3/sqlite3.h @@ -23,7 +23,7 @@ ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. +** on how SQLite interfaces are supposed to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting @@ -43,21 +43,25 @@ extern "C" { /* -** Add the ability to override 'extern' +** Provide the ability to override linkage features of the interface. */ #ifndef SQLITE_EXTERN # define SQLITE_EXTERN extern #endif - #ifndef SQLITE_API # define SQLITE_API #endif - +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif +#ifndef SQLITE_STDCALL +# define SQLITE_STDCALL +#endif /* ** These no-op macros are used in front of interfaces to mark those ** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards +** should not use deprecated interfaces - they are supported for backwards ** compatibility only. Application writers should be aware that ** experimental interfaces are subject to change in point releases. ** @@ -107,9 +111,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.7.2" -#define SQLITE_VERSION_NUMBER 3008007 -#define SQLITE_SOURCE_ID "2014-11-18 20:57:56 2ab564bf9655b7c7b97ab85cafc8a48329b27f93" +#define SQLITE_VERSION "3.9.2" +#define SQLITE_VERSION_NUMBER 3009002 +#define SQLITE_SOURCE_ID "2015-11-02 18:31:45 bda77dda9697c463c3d0704014d51627fceee328" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -120,7 +124,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** **
@@ -142,9 +146,9 @@ extern "C" {
 ** See also: [sqlite_version()] and [sqlite_source_id()].
 */
 SQLITE_API SQLITE_EXTERN const char sqlite3_version[];
-SQLITE_API const char *sqlite3_libversion(void);
-SQLITE_API const char *sqlite3_sourceid(void);
-SQLITE_API int sqlite3_libversion_number(void);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void);
+SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void);
 
 /*
 ** CAPI3REF: Run-Time Library Compilation Options Diagnostics
@@ -169,8 +173,8 @@ SQLITE_API int sqlite3_libversion_number(void);
 ** [sqlite_compileoption_get()] and the [compile_options pragma].
 */
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
-SQLITE_API int sqlite3_compileoption_used(const char *zOptName);
-SQLITE_API const char *sqlite3_compileoption_get(int N);
+SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N);
 #endif
 
 /*
@@ -201,7 +205,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
 ** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but
 ** can be fully or partially disabled using a call to [sqlite3_config()]
 ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD],
-** or [SQLITE_CONFIG_MUTEX].  ^(The return value of the
+** or [SQLITE_CONFIG_SERIALIZED].  ^(The return value of the
 ** sqlite3_threadsafe() function shows only the compile-time setting of
 ** thread safety, not any run-time changes to that setting made by
 ** sqlite3_config(). In other words, the return value from sqlite3_threadsafe()
@@ -209,7 +213,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
 **
 ** See the [threading mode] documentation for additional information.
 */
-SQLITE_API int sqlite3_threadsafe(void);
+SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void);
 
 /*
 ** CAPI3REF: Database Connection Handle
@@ -266,6 +270,7 @@ typedef sqlite_uint64 sqlite3_uint64;
 
 /*
 ** CAPI3REF: Closing A Database Connection
+** DESTRUCTOR: sqlite3
 **
 ** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors
 ** for the [sqlite3] object.
@@ -305,8 +310,8 @@ typedef sqlite_uint64 sqlite3_uint64;
 ** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer
 ** argument is a harmless no-op.
 */
-SQLITE_API int sqlite3_close(sqlite3*);
-SQLITE_API int sqlite3_close_v2(sqlite3*);
+SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3*);
+SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3*);
 
 /*
 ** The type for a callback function.
@@ -317,6 +322,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 
 /*
 ** CAPI3REF: One-Step Query Execution Interface
+** METHOD: sqlite3
 **
 ** The sqlite3_exec() interface is a convenience wrapper around
 ** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()],
@@ -368,7 +374,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
    -**
  • The application must insure that the 1st parameter to sqlite3_exec() +**
  • The application must ensure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. **
  • The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. @@ -376,7 +382,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. **
*/ -SQLITE_API int sqlite3_exec( +SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ @@ -471,6 +477,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) @@ -756,14 +763,16 @@ struct sqlite3_io_methods { ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] ** interface. ** +**
    +**
  • [[SQLITE_FCNTL_LOCKSTATE]] ** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) ** into an integer that the pArg argument points to. This capability -** is used during testing and only needs to be supported when SQLITE_TEST -** is defined. -**
      +** is used during testing and is only available when the SQLITE_TEST +** compile-time option is used. +** **
    • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the @@ -888,7 +897,9 @@ struct sqlite3_io_methods { ** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] ** file control returns [SQLITE_OK], then the parser assumes that the ** VFS has handled the PRAGMA itself and the parser generates a no-op -** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** prepared statement if result string is NULL, or that returns a copy +** of the result string if the string is non-NULL. +** ^If the [SQLITE_FCNTL_PRAGMA] file control returns ** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means ** that the VFS encountered an error while handling the [PRAGMA] and the ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] @@ -946,12 +957,27 @@ struct sqlite3_io_methods { ** pointed to by the pArg argument. This capability is used during testing ** and only needs to be supported when SQLITE_TEST is defined. ** +**
    • [[SQLITE_FCNTL_WAL_BLOCK]] +** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might +** be advantageous to block on the next WAL lock if the lock is not immediately +** available. The WAL subsystem issues this signal during rare +** circumstances in order to fix a problem with priority inversion. +** Applications should not use this file-control. +** +**
    • [[SQLITE_FCNTL_ZIPVFS]] +** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other +** VFS should return SQLITE_NOTFOUND for this opcode. +** +**
    • [[SQLITE_FCNTL_RBU]] +** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by +** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for +** this opcode. **
    */ #define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 @@ -970,6 +996,15 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_WAL_BLOCK 24 +#define SQLITE_FCNTL_ZIPVFS 25 +#define SQLITE_FCNTL_RBU 26 + +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO + /* ** CAPI3REF: Mutex Handle @@ -1221,7 +1256,7 @@ struct sqlite3_vfs { **
** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED @@ -1318,10 +1353,10 @@ struct sqlite3_vfs { ** must return [SQLITE_OK] on success and some other [error code] upon ** failure. */ -SQLITE_API int sqlite3_initialize(void); -SQLITE_API int sqlite3_shutdown(void); -SQLITE_API int sqlite3_os_init(void); -SQLITE_API int sqlite3_os_end(void); +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void); +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); /* ** CAPI3REF: Configuring The SQLite Library @@ -1332,9 +1367,11 @@ SQLITE_API int sqlite3_os_end(void); ** applications and so this routine is usually not necessary. It is ** provided to support rare applications with unusual needs. ** -** The sqlite3_config() interface is not threadsafe. The application -** must insure that no other SQLite interfaces are invoked by other -** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. +** +** The sqlite3_config() interface ** may only be invoked prior to library initialization using ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before @@ -1352,10 +1389,11 @@ SQLITE_API int sqlite3_os_end(void); ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. */ -SQLITE_API int sqlite3_config(int, ...); +SQLITE_API int SQLITE_CDECL sqlite3_config(int, ...); /* ** CAPI3REF: Configure database connections +** METHOD: sqlite3 ** ** The sqlite3_db_config() interface is used to make configuration ** changes to a [database connection]. The interface is similar to @@ -1370,7 +1408,7 @@ SQLITE_API int sqlite3_config(int, ...); ** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if ** the call is considered successful. */ -SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); /* ** CAPI3REF: Memory Allocation Routines @@ -1504,31 +1542,33 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option. ** ** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The argument specifies +**
^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies ** alternative low-level memory allocation routines to be used in place of ** the memory allocation routines built into SQLite.)^ ^SQLite makes ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** ** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +**
^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** ** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
-**
^This option takes single argument of type int, interpreted as a -** boolean, which enables or disables the collection of memory allocation -** statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +**
^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
    **
  • [sqlite3_memory_used()] **
  • [sqlite3_memory_highwater()] **
  • [sqlite3_soft_heap_limit64()] -**
  • [sqlite3_status()] +**
  • [sqlite3_status64()] **
)^ ** ^Memory allocation statistics are enabled by default unless SQLite is ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory @@ -1536,53 +1576,67 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
-**
^This option specifies a static memory buffer that SQLite can use for -** scratch memory. There are three arguments: A pointer an 8-byte +**
^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer +** that SQLite can use for scratch memory. ^(There are three arguments +** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. +** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than one scratch buffers per thread. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.
+** [sqlite3_malloc()] will be used to obtain the memory needed.

+** ^When the application provides any amount of scratch memory using +** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large +** [sqlite3_malloc|heap allocations]. +** This can help [Robson proof|prevent memory allocation failures] due to heap +** fragmentation in low-memory embedded systems. +** ** ** [[SQLITE_CONFIG_PAGECACHE]]

SQLITE_CONFIG_PAGECACHE
-**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implementation. +**
^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** that SQLite can use for the database page cache with the default page +** cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. -** There are three arguments to this option: A pointer to 8-byte aligned +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus a little extra for each -** page header. ^The page header size is 20 to 40 bytes depending on -** the host architecture. ^It is harmless, apart from the wasted memory, -** to make sz a little too large. The first -** argument should point to an allocation of at least sz*N bytes of memory. +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The pointer in the first argument must -** be aligned to an 8-byte boundary or subsequent behavior of SQLite -** will be undefined.
+** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
-**
^This option specifies a static memory buffer that SQLite will use -** for all of its dynamic memory allocation needs beyond those provided -** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** There are three arguments: An 8-byte aligned pointer to the memory, +**
^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the -** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or -** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** memory pointer is not NULL then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. @@ -1590,11 +1644,11 @@ struct sqlite3_mem_methods { ** for the minimum allocation size are 2**5 through 2**8.
** ** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The argument specifies -** alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +**
^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to @@ -1602,8 +1656,8 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
** ** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The +**
^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] ** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation @@ -1615,25 +1669,25 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
** ** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
-**
^(This option takes two arguments that determine the default -** memory allocation for the lookaside memory allocator on each -** [database connection]. The first argument is the +**
^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** verb to [sqlite3_db_config()] can be used to change the lookaside +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** ** [[SQLITE_CONFIG_PCACHE2]]
SQLITE_CONFIG_PCACHE2
-**
^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods2] object. This object specifies the interface -** to a custom page cache implementation.)^ ^SQLite makes a copy of the -** object and uses it for page cache memory allocations.
+**
^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
** ** [[SQLITE_CONFIG_GETPCACHE2]]
SQLITE_CONFIG_GETPCACHE2
-**
^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
+**
^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
** ** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
The SQLITE_CONFIG_LOG option is used to configure the SQLite @@ -1656,10 +1710,11 @@ struct sqlite3_mem_methods { ** function must be threadsafe.
** ** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI -**
^(This option takes a single argument of type int. If non-zero, then -** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled.)^ ^If URI handling is globally enabled, all filenames -** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +**
^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1669,9 +1724,10 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined.)^ ** ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
SQLITE_CONFIG_COVERING_INDEX_SCAN -**
^This option takes a single integer argument which is interpreted as -** a boolean in order to enable or disable the use of covering indices for -** full table scans in the query optimizer. ^The default setting is determined +**
^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" ** if that compile-time option is omitted. ** The ability to disable the use of covering indices for full table scans @@ -1711,18 +1767,37 @@ struct sqlite3_mem_methods { ** ^The default setting can be overridden by each database connection using ** either the [PRAGMA mmap_size] command, or by using the ** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size -** cannot be changed at run-time. Nor may the maximum allowed mmap size -** exceed the compile-time maximum mmap size set by the +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ ** ^If either argument to this option is negative, then that argument is ** changed to its compile-time default. ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
SQLITE_CONFIG_WIN32_HEAPSIZE -**
^This option is only available if SQLite is compiled for Windows -** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +**
^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
SQLITE_CONFIG_PCACHE_HDRSZ +**
^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** +** [[SQLITE_CONFIG_PMASZ]] +**
SQLITE_CONFIG_PMASZ +**
^The SQLITE_CONFIG_PMASZ option takes a single parameter which +** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded +** sorter to that integer. The default minimum PMA Size is set by the +** [SQLITE_SORTER_PMASZ] compile-time option. New threads are launched +** to help with sort operations when multithreaded sorting +** is enabled (using the [PRAGMA threads] command) and the amount of content +** to be sorted exceeds the page size times the minimum of the +** [PRAGMA cache_size] setting and this value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1748,6 +1823,8 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ +#define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1814,15 +1891,17 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Enable Or Disable Extended Result Codes +** METHOD: sqlite3 ** ** ^The sqlite3_extended_result_codes() routine enables or disables the ** [extended result codes] feature of SQLite. ^The extended result ** codes are disabled by default for historical compatibility. */ -SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid +** METHOD: sqlite3 ** ** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) ** has a unique 64-bit signed @@ -1870,52 +1949,51 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** unpredictable and might not equal either the old or the new ** last insert [rowid]. */ -SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified +** METHOD: sqlite3 +** +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. -** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. -** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. +** +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
    +**
  • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
  • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
+** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -1924,25 +2002,23 @@ SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified +** METHOD: sqlite3 ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** @@ -1950,10 +2026,11 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); /* ** CAPI3REF: Interrupt A Long-Running Query +** METHOD: sqlite3 ** ** ^This function causes any pending database operation to abort and ** return at its earliest opportunity. This routine is typically @@ -1989,7 +2066,7 @@ SQLITE_API int sqlite3_total_changes(sqlite3*); ** If the database connection closes while [sqlite3_interrupt()] ** is running then bad things will likely happen. */ -SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -2024,11 +2101,13 @@ SQLITE_API void sqlite3_interrupt(sqlite3*); ** The input to [sqlite3_complete16()] must be a zero-terminated ** UTF-16 string in native byte order. */ -SQLITE_API int sqlite3_complete(const char *sql); -SQLITE_API int sqlite3_complete16(const void *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} +** METHOD: sqlite3 ** ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X ** that might be invoked with argument P whenever @@ -2045,7 +2124,7 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for the same locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] is returned ** to the application. @@ -2084,10 +2163,11 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); /* ** CAPI3REF: Set A Busy Timeout +** METHOD: sqlite3 ** ** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps ** for a specified amount of time when a table is locked. ^The handler @@ -2106,10 +2186,11 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** ** See also: [PRAGMA busy_timeout] */ -SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries +** METHOD: sqlite3 ** ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. @@ -2180,7 +2261,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** reflected in subsequent calls to [sqlite3_errcode()] or ** [sqlite3_errmsg()]. */ -SQLITE_API int sqlite3_get_table( +SQLITE_API int SQLITE_STDCALL sqlite3_get_table( sqlite3 *db, /* An open database */ const char *zSql, /* SQL to be evaluated */ char ***pazResult, /* Results of the query */ @@ -2188,13 +2269,17 @@ SQLITE_API int sqlite3_get_table( int *pnColumn, /* Number of result columns written here */ char **pzErrmsg /* Error msg written here */ ); -SQLITE_API void sqlite3_free_table(char **result); +SQLITE_API void SQLITE_STDCALL sqlite3_free_table(char **result); /* ** CAPI3REF: Formatted String Printing Functions ** ** These routines are work-alikes of the "printf()" family of functions ** from the standard C library. +** These routines understand most of the common K&R formatting options, +** plus some additional non-standard formats, detailed below. +** Note that some of the more obscure formatting options from recent +** C-library standards are omitted from this implementation. ** ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their ** results into memory obtained from [sqlite3_malloc()]. @@ -2227,7 +2312,7 @@ SQLITE_API void sqlite3_free_table(char **result); ** These routines all implement some additional formatting ** options that are useful for constructing SQL statements. ** All of the usual printf() formatting options apply. In addition, there -** is are "%q", "%Q", and "%z" options. +** is are "%q", "%Q", "%w" and "%z" options. ** ** ^(The %q option works like %s in that it substitutes a nul-terminated ** string from the argument list. But %q also doubles every '\'' character. @@ -2280,14 +2365,20 @@ SQLITE_API void sqlite3_free_table(char **result); ** The code above will render a correct SQL statement in the zSQL ** variable even if the zText variable is a NULL pointer. ** +** ^(The "%w" formatting option is like "%q" except that it expects to +** be contained within double-quotes instead of single quotes, and it +** escapes the double-quote character instead of the single-quote +** character.)^ The "%w" formatting option is intended for safely inserting +** table and column names into a constructed SQL statement. +** ** ^(The "%z" formatting option works like "%s" but with the ** addition that after the string has been read and copied into ** the result, [sqlite3_free()] is called on the input string.)^ */ -SQLITE_API char *sqlite3_mprintf(const char*,...); -SQLITE_API char *sqlite3_vmprintf(const char*, va_list); -SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); -SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char*,...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list); /* ** CAPI3REF: Memory Allocation Subsystem @@ -2377,12 +2468,12 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** a block of memory after it has been released using ** [sqlite3_free()] or [sqlite3_realloc()]. */ -SQLITE_API void *sqlite3_malloc(int); -SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); -SQLITE_API void *sqlite3_realloc(void*, int); -SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); -SQLITE_API void sqlite3_free(void*); -SQLITE_API sqlite3_uint64 sqlite3_msize(void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void*, int); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void SQLITE_STDCALL sqlite3_free(void*); +SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2407,8 +2498,8 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** by [sqlite3_memory_highwater(1)] is the high-water mark ** prior to the reset. */ -SQLITE_API sqlite3_int64 sqlite3_memory_used(void); -SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag); /* ** CAPI3REF: Pseudo-Random Number Generator @@ -2420,20 +2511,22 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ -SQLITE_API void sqlite3_randomness(int N, void *P); +SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P); /* ** CAPI3REF: Compile-Time Authorization Callbacks +** METHOD: sqlite3 ** ** ^This routine registers an authorizer callback with a particular ** [database connection], supplied in the first argument. @@ -2512,7 +2605,7 @@ SQLITE_API void sqlite3_randomness(int N, void *P); ** as stated in the previous paragraph, sqlite3_step() invokes ** sqlite3_prepare_v2() to reprepare a statement after a schema change. */ -SQLITE_API int sqlite3_set_authorizer( +SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3*, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pUserData @@ -2590,6 +2683,7 @@ SQLITE_API int sqlite3_set_authorizer( /* ** CAPI3REF: Tracing And Profiling Functions +** METHOD: sqlite3 ** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. @@ -2616,12 +2710,13 @@ SQLITE_API int sqlite3_set_authorizer( ** sqlite3_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ -SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); -SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); /* ** CAPI3REF: Query Progress Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to @@ -2651,10 +2746,11 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, ** database connections for the meaning of "modify" in this paragraph. ** */ -SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection +** CONSTRUCTOR: sqlite3 ** ** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for @@ -2879,15 +2975,15 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** See also: [sqlite3_temp_directory] */ -SQLITE_API int sqlite3_open( +SQLITE_API int SQLITE_STDCALL sqlite3_open( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open16( +SQLITE_API int SQLITE_STDCALL sqlite3_open16( const void *filename, /* Database filename (UTF-16) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb, /* OUT: SQLite db handle */ int flags, /* Flags */ @@ -2933,19 +3029,22 @@ SQLITE_API int sqlite3_open_v2( ** VFS method, then the behavior of this routine is undefined and probably ** undesirable. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* ** CAPI3REF: Error Codes And Messages -** -** ^The sqlite3_errcode() interface returns the numeric [result code] or -** [extended result code] for the most recent failed sqlite3_* API call -** associated with a [database connection]. If a prior API call failed -** but the most recent API call succeeded, the return value from -** sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() +** METHOD: sqlite3 +** +** ^If the most recent sqlite3_* API call associated with +** [database connection] D failed, then the sqlite3_errcode(D) interface +** returns the numeric [result code] or [extended result code] for that +** API call. +** If the most recent API call was successful, +** then the return value from sqlite3_errcode() is undefined. +** ^The sqlite3_extended_errcode() ** interface is the same except that it always returns the ** [extended result code] even when extended result codes are ** disabled. @@ -2976,40 +3075,41 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** was invoked incorrectly by the application. In that case, the ** error code and message may or may not be set. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db); -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); -SQLITE_API const char *sqlite3_errmsg(sqlite3*); -SQLITE_API const void *sqlite3_errmsg16(sqlite3*); -SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int); /* -** CAPI3REF: SQL Statement Object +** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} ** -** An instance of this object represents a single SQL statement. -** This object is variously known as a "prepared statement" or a -** "compiled SQL statement" or simply as a "statement". +** An instance of this object represents a single SQL statement that +** has been compiled into binary form and is ready to be evaluated. +** +** Think of each SQL statement as a separate computer program. The +** original SQL text is source code. A prepared statement object +** is the compiled object code. All SQL must be converted into a +** prepared statement before it can be run. ** -** The life of a statement object goes something like this: +** The life-cycle of a prepared statement object usually goes like this: ** **
    -**
  1. Create the object using [sqlite3_prepare_v2()] or a related -** function. -**
  2. Bind values to [host parameters] using the sqlite3_bind_*() +**
  3. Create the prepared statement object using [sqlite3_prepare_v2()]. +**
  4. Bind values to [parameters] using the sqlite3_bind_*() ** interfaces. **
  5. Run the SQL by calling [sqlite3_step()] one or more times. -**
  6. Reset the statement using [sqlite3_reset()] then go back +**
  7. Reset the prepared statement using [sqlite3_reset()] then go back ** to step 2. Do this zero or more times. **
  8. Destroy the object using [sqlite3_finalize()]. **
-** -** Refer to documentation on individual methods above for additional -** information. */ typedef struct sqlite3_stmt sqlite3_stmt; /* ** CAPI3REF: Run-time Limits +** METHOD: sqlite3 ** ** ^(This interface allows the size of various constructs to be limited ** on a connection by connection basis. The first parameter is the @@ -3047,7 +3147,7 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** ** New run-time limit categories may be added in future releases. */ -SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Run-Time Limit Categories @@ -3121,6 +3221,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Compiling An SQL Statement ** KEYWORDS: {SQL statement compiler} +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_stmt ** ** To execute an SQL query, it must first be compiled into a byte-code ** program using one of these routines. @@ -3134,16 +3236,14 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() ** use UTF-16. ** -** ^If the nByte argument is less than zero, then zSql is read up to the -** first zero terminator. ^If nByte is non-negative, then it is the maximum -** number of bytes read from zSql. ^When nByte is non-negative, the -** zSql string ends at either the first '\000' or '\u0000' character or -** the nByte-th byte, whichever comes first. If the caller knows -** that the supplied string is nul-terminated, then there is a small -** performance advantage to be gained by passing an nByte parameter that -** is equal to the number of bytes in the input string including -** the nul-terminator bytes as this saves SQLite from having to -** make a copy of the input string. +** ^If the nByte argument is negative, then zSql is read up to the +** first zero terminator. ^If nByte is positive, then it is the +** number of bytes read from zSql. ^If nByte is zero, then no prepared +** statement is generated. +** If the caller knows that the supplied string is nul-terminated, then +** there is a small performance advantage to passing an nByte parameter that +** is the number of bytes in the input string including +** the nul-terminator. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -3199,28 +3299,28 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** */ -SQLITE_API int sqlite3_prepare( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const void **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ @@ -3230,15 +3330,17 @@ SQLITE_API int sqlite3_prepare16_v2( /* ** CAPI3REF: Retrieving Statement SQL +** METHOD: sqlite3_stmt ** ** ^This interface can be used to retrieve a saved copy of the original ** SQL text used to create a [prepared statement] if that statement was ** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. */ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database +** METHOD: sqlite3_stmt ** ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if ** and only if the [prepared statement] X makes no direct changes to @@ -3266,14 +3368,16 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** change the configuration of a database connection, they do not make ** changes to the content of the database files on disk. */ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** METHOD: sqlite3_stmt ** ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the ** [prepared statement] S has been stepped at least once using -** [sqlite3_step(S)] but has not run to completion and/or has not +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor ** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) ** interface returns false if S is a NULL pointer. If S is not a ** NULL pointer and is not a pointer to a valid [prepared statement] @@ -3285,7 +3389,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** for example, in diagnostic routines to search for prepared ** statements that are holding a transaction open. */ -SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*); /* ** CAPI3REF: Dynamically Typed Value Object @@ -3300,7 +3404,9 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); ** Some interfaces require a protected sqlite3_value. Other interfaces ** will accept either a protected or an unprotected sqlite3_value. ** Every interface that accepts sqlite3_value arguments specifies -** whether or not it requires a protected sqlite3_value. +** whether or not it requires a protected sqlite3_value. The +** [sqlite3_value_dup()] interface can be used to construct a new +** protected sqlite3_value from an unprotected sqlite3_value. ** ** The terms "protected" and "unprotected" refer to whether or not ** a mutex is held. An internal mutex is held for a protected @@ -3344,6 +3450,7 @@ typedef struct sqlite3_context sqlite3_context; ** CAPI3REF: Binding Values To Prepared Statements ** KEYWORDS: {host parameter} {host parameters} {host parameter name} ** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, ** literals may be replaced by a [parameter] that matches one of following @@ -3446,22 +3553,24 @@ typedef struct sqlite3_context sqlite3_context; ** See also: [sqlite3_bind_parameter_count()], ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); -SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, void(*)(void*)); -SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); -SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); -SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); -SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); -SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); -SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); /* ** CAPI3REF: Number Of SQL Parameters +** METHOD: sqlite3_stmt ** ** ^This routine can be used to find the number of [SQL parameters] ** in a [prepared statement]. SQL parameters are tokens of the @@ -3478,10 +3587,11 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); ** [sqlite3_bind_parameter_name()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt*); /* ** CAPI3REF: Name Of A Host Parameter +** METHOD: sqlite3_stmt ** ** ^The sqlite3_bind_parameter_name(P,N) interface returns ** the name of the N-th [SQL parameter] in the [prepared statement] P. @@ -3505,10 +3615,11 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); ** [sqlite3_bind_parameter_count()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*, int); /* ** CAPI3REF: Index Of A Parameter With A Given Name +** METHOD: sqlite3_stmt ** ** ^Return the index of an SQL parameter given its name. ^The ** index value returned is suitable for use as the second @@ -3519,21 +3630,23 @@ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); ** ** See also: [sqlite3_bind_blob|sqlite3_bind()], ** [sqlite3_bind_parameter_count()], and -** [sqlite3_bind_parameter_index()]. +** [sqlite3_bind_parameter_name()]. */ -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); /* ** CAPI3REF: Reset All Bindings On A Prepared Statement +** METHOD: sqlite3_stmt ** ** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset ** the [sqlite3_bind_blob | bindings] on a [prepared statement]. ** ^Use this routine to reset all host parameters to NULL. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt*); /* ** CAPI3REF: Number Of Columns In A Result Set +** METHOD: sqlite3_stmt ** ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL @@ -3541,10 +3654,11 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ** See also: [sqlite3_data_count()] */ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set +** METHOD: sqlite3_stmt ** ** ^These routines return the name assigned to a particular column ** in the result set of a [SELECT] statement. ^The sqlite3_column_name() @@ -3569,11 +3683,12 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); ** then the name of the column is unspecified and may change from ** one release of SQLite to the next. */ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N); /* ** CAPI3REF: Source Of Data In A Query Result +** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and ** table column that is the origin of a particular result column in @@ -3617,15 +3732,16 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** for the same [prepared statement] and result column ** at the same time then the results are undefined. */ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt*,int); /* ** CAPI3REF: Declared Datatype Of A Query Result +** METHOD: sqlite3_stmt ** ** ^(The first parameter is a [prepared statement]. ** If this statement is a [SELECT] statement and the Nth column of the @@ -3653,11 +3769,12 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); ** is associated with individual values, not with the containers ** used to hold those values. */ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,int); /* ** CAPI3REF: Evaluate An SQL Statement +** METHOD: sqlite3_stmt ** ** After a [prepared statement] has been prepared using either ** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy @@ -3733,10 +3850,11 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** then the more specific [error codes] are returned directly ** by sqlite3_step(). The use of the "v2" interface is recommended. */ -SQLITE_API int sqlite3_step(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set +** METHOD: sqlite3_stmt ** ** ^The sqlite3_data_count(P) interface returns the number of columns in the ** current row of the result set of [prepared statement] P. @@ -3753,7 +3871,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** ** See also: [sqlite3_column_count()] */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes @@ -3790,8 +3908,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Result Values From A Query ** KEYWORDS: {column access functions} -** -** These routines form the "result set" interface. +** METHOD: sqlite3_stmt ** ** ^These routines return information about a single column of the current ** result row of a query. ^In every case the first argument is a pointer @@ -3852,13 +3969,14 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** -** ^The object returned by [sqlite3_column_value()] is an -** [unprotected sqlite3_value] object. An unprotected sqlite3_value object -** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by ** [sqlite3_column_value()] is used in any other way, including calls ** to routines like [sqlite3_value_int()], [sqlite3_value_text()], -** or [sqlite3_value_bytes()], then the behavior is undefined. +** or [sqlite3_value_bytes()], the behavior is not threadsafe. ** ** These routines attempt to convert the value where appropriate. ^For ** example, if the internal representation is FLOAT and a text result @@ -3889,12 +4007,6 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** **
)^ ** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** own equivalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. -** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. @@ -3919,7 +4031,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** **
    @@ -3939,7 +4051,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** @@ -3949,19 +4061,20 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** pointer. Subsequent calls to [sqlite3_errcode()] will return ** [SQLITE_NOMEM].)^ */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); -SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors @@ -3985,10 +4098,11 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** statement after it has been finalized can result in undefined and ** undesirable behavior such as segfaults and heap corruption. */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt ** ** The sqlite3_reset() function is called to reset a [prepared statement] ** object back to its initial state, ready to be re-executed. @@ -4011,13 +4125,14 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} +** METHOD: sqlite3 ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior @@ -4110,7 +4225,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. */ -SQLITE_API int sqlite3_create_function( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4120,7 +4235,7 @@ SQLITE_API int sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( sqlite3 *db, const void *zFunctionName, int nArg, @@ -4130,7 +4245,7 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4148,9 +4263,9 @@ SQLITE_API int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ @@ -4172,25 +4287,26 @@ SQLITE_API int sqlite3_create_function_v2( ** These functions are [deprecated]. In order to maintain ** backwards compatibility with older code, these functions continue ** to be supported. However, new applications should avoid -** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you what they do. +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. */ #ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); -SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), void*,sqlite3_int64); #endif /* -** CAPI3REF: Obtaining SQL Function Parameter Values +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on -** the function or aggregate. +** the function or aggregate. ** ** The xFunc (for scalar functions) or xStep (for aggregates) parameters ** to [sqlite3_create_function()] and [sqlite3_create_function16()] @@ -4230,21 +4346,55 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); -SQLITE_API double sqlite3_value_double(sqlite3_value*); -SQLITE_API int sqlite3_value_int(sqlite3_value*); -SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); -SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); -SQLITE_API int sqlite3_value_type(sqlite3_value*); -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value*); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); + +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** SQLite makes no use of subtype itself. It merely passes the subtype +** from the result of one [application-defined SQL function] into the +** input of another. +*/ +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API SQLITE_EXPERIMENTAL sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value*); +SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context ** ** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. @@ -4285,10 +4435,11 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** This routine must be called from the same thread in which ** the aggregate SQL function is running. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); +SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_user_data() interface returns a copy of ** the pointer that was the pUserData parameter (the 5th parameter) @@ -4299,10 +4450,11 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); ** This routine must be called from the same thread in which ** the application-defined function is running. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context*); +SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context*); /* ** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_context_db_handle() interface returns a copy of ** the pointer to the [database connection] (the 1st parameter) @@ -4310,10 +4462,11 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context*); ** and [sqlite3_create_function16()] routines that originally ** registered the application defined function. */ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to ** associate metadata with argument values. If the same value is passed to @@ -4362,8 +4515,8 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** These routines must be called from the same thread in which ** the SQL function is running. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); -SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); /* @@ -4386,6 +4539,7 @@ typedef void (*sqlite3_destructor_type)(void*); /* ** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See @@ -4401,9 +4555,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite3_result_zeroblob() interfaces set the result of -** the application-defined function to be a BLOB containing all zero -** bytes and N bytes in size, where N is the value of the 2nd parameter. +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. ** ** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified @@ -4485,7 +4639,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** from [sqlite3_malloc()] before it returns. ** ** ^The sqlite3_result_value() interface sets the result of -** the application-defined function to be a copy the +** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The ** sqlite3_result_value() interface makes a copy of the [sqlite3_value] ** so that the [sqlite3_value] specified in the parameter may change or @@ -4498,28 +4652,46 @@ typedef void (*sqlite3_destructor_type)(void*); ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ -SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); -SQLITE_API void sqlite3_result_double(sqlite3_context*, double); -SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); -SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); -SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); -SQLITE_API void sqlite3_result_null(sqlite3_context*); -SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned int); /* ** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 ** ** ^These functions add, remove, or modify a [collation] associated ** with the [database connection] specified as the first argument. @@ -4597,14 +4769,14 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ -SQLITE_API int sqlite3_create_collation( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); -SQLITE_API int sqlite3_create_collation_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, @@ -4612,7 +4784,7 @@ SQLITE_API int sqlite3_create_collation_v2( int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); -SQLITE_API int sqlite3_create_collation16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, @@ -4622,6 +4794,7 @@ SQLITE_API int sqlite3_create_collation16( /* ** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 ** ** ^To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the @@ -4646,12 +4819,12 @@ SQLITE_API int sqlite3_create_collation16( ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. */ -SQLITE_API int sqlite3_collation_needed( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const char*) ); -SQLITE_API int sqlite3_collation_needed16( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const void*) @@ -4665,11 +4838,11 @@ SQLITE_API int sqlite3_collation_needed16( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_key( +SQLITE_API int SQLITE_STDCALL sqlite3_key( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The key */ ); -SQLITE_API int sqlite3_key_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_key_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The key */ @@ -4683,11 +4856,11 @@ SQLITE_API int sqlite3_key_v2( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_rekey( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The new key */ ); -SQLITE_API int sqlite3_rekey_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The new key */ @@ -4697,7 +4870,7 @@ SQLITE_API int sqlite3_rekey_v2( ** Specify the activation key for a SEE database. Unless ** activated, none of the SEE routines will work. */ -SQLITE_API void sqlite3_activate_see( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_see( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -4707,7 +4880,7 @@ SQLITE_API void sqlite3_activate_see( ** Specify the activation key for a CEROD database. Unless ** activated, none of the CEROD routines will work. */ -SQLITE_API void sqlite3_activate_cerod( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_cerod( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -4729,7 +4902,7 @@ SQLITE_API void sqlite3_activate_cerod( ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. */ -SQLITE_API int sqlite3_sleep(int); +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -4829,6 +5002,7 @@ SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 ** ** ^The sqlite3_get_autocommit() interface returns non-zero or ** zero if the given database connection is or is not in autocommit mode, @@ -4847,10 +5021,11 @@ SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; ** connection while this routine is running, then the return value ** is undefined. */ -SQLITE_API int sqlite3_get_autocommit(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3*); /* ** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt ** ** ^The sqlite3_db_handle interface returns the [database connection] handle ** to which a [prepared statement] belongs. ^The [database connection] @@ -4859,10 +5034,11 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); ** to the [sqlite3_prepare_v2()] call (or its variants) that was used to ** create the statement in the first place. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt*); /* ** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename ** associated with database N of connection D. ^The main database file @@ -4875,19 +5051,21 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** will be an absolute pathname, even if the filename used ** to open the database originally was a URI or relative pathname. */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 ** ** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N ** of connection D is read-only, 0 if it is read/write, or -1 if N is not ** the name of a database on connection D. */ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); +SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 ** ** ^This interface returns a pointer to the next [prepared statement] after ** pStmt associated with the [database connection] pDb. ^If pStmt is NULL @@ -4899,10 +5077,11 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); ** [sqlite3_next_stmt(D,S)] must refer to an open database ** connection and in particular must not be a NULL pointer. */ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); +SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_commit_hook() interface registers a callback ** function to be invoked whenever a transaction is [COMMIT | committed]. @@ -4947,11 +5126,12 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** ** See also the [sqlite3_update_hook()] interface. */ -SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); -SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* ** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument @@ -4998,7 +5178,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] ** interfaces. */ -SQLITE_API void *sqlite3_update_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( sqlite3*, void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* @@ -5028,12 +5208,17 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. +** ** This interface is threadsafe on processors where writing a ** 32-bit integer is atomic. ** ** See Also: [SQLite Shared-Cache Mode] */ -SQLITE_API int sqlite3_enable_shared_cache(int); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int); /* ** CAPI3REF: Attempt To Free Heap Memory @@ -5049,10 +5234,11 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** ** See also: [sqlite3_db_release_memory()] */ -SQLITE_API int sqlite3_release_memory(int); +SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int); /* ** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap ** memory as possible from database connection D. Unlike the @@ -5062,7 +5248,7 @@ SQLITE_API int sqlite3_release_memory(int); ** ** See also: [sqlite3_release_memory()] */ -SQLITE_API int sqlite3_db_release_memory(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); /* ** CAPI3REF: Impose A Limit On Heap Size @@ -5114,7 +5300,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** The circumstances under which SQLite will enforce the soft heap limit may ** changes in future releases of SQLite. */ -SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 N); /* ** CAPI3REF: Deprecated Soft Heap Limit Interface @@ -5125,26 +5311,34 @@ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); ** only. All new applications should use the ** [sqlite3_soft_heap_limit64()] interface rather than this one. */ -SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table -** -** ^This routine returns metadata about a specific column of a specific -** database table accessible using the [database connection] handle -** passed as the first function argument. +** METHOD: sqlite3 +** +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR and if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existance of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. ** ** ^The column is identified by the second, third and fourth parameters to -** this function. ^The second parameter is either the name of the database +** this function. ^(The second parameter is either the name of the database ** (i.e. "main", "temp", or an attached database) containing the specified -** table or NULL. ^If it is NULL, then all attached databases are searched +** table or NULL.)^ ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** ** ^The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** name of the desired column, respectively. ** ** ^Metadata is returned by writing to the memory locations passed as the 5th ** and subsequent parameters to this function. ^Any of these arguments may be @@ -5163,16 +5357,17 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** )^ ** ** ^The memory pointed to by the character pointers returned for the -** declaration type and collation sequence is valid only until the next +** declaration type and collation sequence is valid until the next ** call to any SQLite API function. ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no -** explicitly declared [INTEGER PRIMARY KEY] column, then the output -** parameters are set as follows: +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: ** **
     **     data type: "INTEGER"
    @@ -5182,15 +5377,11 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
     **     auto increment: 0
     ** 
    )^ ** -** ^(This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ -** -** ^This API is only available if the library was compiled with the -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ -SQLITE_API int sqlite3_table_column_metadata( +SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -5204,6 +5395,7 @@ SQLITE_API int sqlite3_table_column_metadata( /* ** CAPI3REF: Load An Extension +** METHOD: sqlite3 ** ** ^This interface loads an SQLite extension library from the named file. ** @@ -5236,7 +5428,7 @@ SQLITE_API int sqlite3_table_column_metadata( ** ** See also the [load_extension() SQL function]. */ -SQLITE_API int sqlite3_load_extension( +SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Derived from zFile if 0 */ @@ -5245,6 +5437,7 @@ SQLITE_API int sqlite3_load_extension( /* ** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 ** ** ^So as not to open security holes in older applications that are ** unprepared to deal with [extension loading], and as a means of disabling @@ -5256,7 +5449,7 @@ SQLITE_API int sqlite3_load_extension( ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* ** CAPI3REF: Automatically Load Statically Linked Extensions @@ -5294,7 +5487,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** See also: [sqlite3_reset_auto_extension()] ** and [sqlite3_cancel_auto_extension()] */ -SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); +SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Cancel Automatic Extension Loading @@ -5306,7 +5499,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); ** unregistered and it returns 0 if X was not on the list of initialization ** routines. */ -SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); +SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading @@ -5314,7 +5507,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); ** ^This interface disables all automatic extensions previously ** registered using [sqlite3_auto_extension()]. */ -SQLITE_API void sqlite3_reset_auto_extension(void); +SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void); /* ** The interface to the virtual-table mechanism is currently considered @@ -5441,13 +5634,31 @@ struct sqlite3_module { ** ^The estimatedRows value is an estimate of the number of rows that ** will be returned by the strategy. ** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info ** structure for SQLite version 3.8.2. If a virtual table extension is ** used with an SQLite version earlier than 3.8.2, the results of attempting ** to read or write the estimatedRows field are undefined (but are likely ** to included crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a -** value greater than or equal to 3008002. +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for version 3.9.0. It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. */ struct sqlite3_index_info { /* Inputs */ @@ -5475,8 +5686,15 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ /* Fields below are only available in SQLite 3.8.2 and later */ sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ }; +/* +** CAPI3REF: Virtual Table Scan Flags +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** @@ -5494,6 +5712,7 @@ struct sqlite3_index_info { /* ** CAPI3REF: Register A Virtual Table Implementation +** METHOD: sqlite3 ** ** ^These routines are used to register a new [virtual table module] name. ** ^Module names must be registered before @@ -5517,13 +5736,13 @@ struct sqlite3_index_info { ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ -SQLITE_API int sqlite3_create_module( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ const sqlite3_module *p, /* Methods for the module */ void *pClientData /* Client data for xCreate/xConnect */ ); -SQLITE_API int sqlite3_create_module_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ const sqlite3_module *p, /* Methods for the module */ @@ -5551,7 +5770,7 @@ SQLITE_API int sqlite3_create_module_v2( */ struct sqlite3_vtab { const sqlite3_module *pModule; /* The module for this virtual table */ - int nRef; /* NO LONGER USED */ + int nRef; /* Number of open cursors */ char *zErrMsg; /* Error message from sqlite3_mprintf() */ /* Virtual table implementations will typically add additional fields */ }; @@ -5586,10 +5805,11 @@ struct sqlite3_vtab_cursor { ** to declare the format (the names and datatypes of the columns) of ** the virtual tables they implement. */ -SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); +SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3*, const char *zSQL); /* ** CAPI3REF: Overload A Function For A Virtual Table +** METHOD: sqlite3 ** ** ^(Virtual tables can provide alternative implementations of functions ** using the [xFindFunction] method of the [virtual table module]. @@ -5604,7 +5824,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); ** purpose is to be a placeholder function that can be overloaded ** by a [virtual table]. */ -SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); /* ** The interface to the virtual-table mechanism defined above (back up @@ -5632,6 +5852,8 @@ typedef struct sqlite3_blob sqlite3_blob; /* ** CAPI3REF: Open A BLOB For Incremental I/O +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_blob ** ** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located ** in row iRow, column zColumn, table zTable in database zDb; @@ -5641,26 +5863,42 @@ typedef struct sqlite3_blob sqlite3_blob; ** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; ** )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. -** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". -** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
      +**
    • ^(Database zDb does not exist)^, +**
    • ^(Table zTable does not exist within database zDb)^, +**
    • ^(Table zTable is a WITHOUT ROWID table)^, +**
    • ^(Column zColumn does not exist)^, +**
    • ^(Row iRow is not present in the table)^, +**
    • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
    • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
    +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5678,18 +5916,14 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. */ -SQLITE_API int sqlite3_blob_open( +SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( sqlite3*, const char *zDb, const char *zTable, @@ -5701,6 +5935,7 @@ SQLITE_API int sqlite3_blob_open( /* ** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob ** ** ^This function is used to move an existing blob handle so that it points ** to a different row of the same database table. ^The new row is identified @@ -5721,34 +5956,34 @@ SQLITE_API int sqlite3_blob_open( ** ** ^This function sets the database handle error code and message. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob ** -** ^Closes an open [BLOB handle]. -** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ -** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ -SQLITE_API int sqlite3_blob_close(sqlite3_blob *); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *); /* ** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The @@ -5760,10 +5995,11 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *); ** been closed by [sqlite3_blob_close()]. Passing any other pointer in ** to this routine results in undefined and probably undesirable behavior. */ -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *); /* ** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob ** ** ^(This function is used to read data from an open [BLOB handle] into a ** caller-supplied buffer. N bytes of data are copied into buffer Z @@ -5788,26 +6024,33 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); ** ** See also: [sqlite3_blob_write()]. */ -SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5816,9 +6059,6 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in @@ -5826,7 +6066,7 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** ** See also: [sqlite3_blob_read()]. */ -SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); /* ** CAPI3REF: Virtual File System Objects @@ -5857,9 +6097,9 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOff ** ^(If the default VFS is unregistered, another VFS is chosen as ** the default. The choice for the new VFS is arbitrary.)^ */ -SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); -SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); -SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); +SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); /* ** CAPI3REF: Mutexes @@ -5871,34 +6111,34 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
      **
    • SQLITE_MUTEX_PTHREADS **
    • SQLITE_MUTEX_W32 **
    • SQLITE_MUTEX_NOOP -**
    )^ +**
** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
    **
  • SQLITE_MUTEX_FAST @@ -5911,7 +6151,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 -**
)^ +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -5919,14 +6163,14 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -5935,16 +6179,13 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -5952,23 +6193,21 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -5976,11 +6215,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ -SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); -SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex*); /* ** CAPI3REF: Mutex Methods Object @@ -5989,9 +6228,9 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -6032,13 +6271,13 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6064,34 +6303,34 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG -SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); #endif /* @@ -6117,9 +6356,13 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ /* ** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 ** ** ^This interface returns a pointer the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument @@ -6127,10 +6370,11 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); ** ^If the [threading mode] is Single-thread or Multi-thread then this ** routine returns a NULL pointer. */ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -6161,7 +6405,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** ** See also: [SQLITE_FCNTL_LOCKSTATE] */ -SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); /* ** CAPI3REF: Testing Interface @@ -6180,7 +6424,7 @@ SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void* ** Unlike most of the SQLite API, this function is not guaranteed to ** operate consistently from one release to the next. */ -SQLITE_API int sqlite3_test_control(int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); /* ** CAPI3REF: Testing Interface Operation Codes @@ -6214,12 +6458,13 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_BYTEORDER 22 #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 -#define SQLITE_TESTCTRL_LAST 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_LAST 25 /* ** CAPI3REF: SQLite Runtime Status ** -** ^This interface is used to retrieve runtime status information +** ^These interfaces are used to retrieve runtime status information ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes @@ -6233,19 +6478,22 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_status() routine returns SQLITE_OK on success and a -** non-zero [error code] on failure. +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. ** -** This routine is threadsafe but is not atomic. This routine can be -** called while other threads are running the same or different SQLite -** interfaces. However the values returned in *pCurrent and -** *pHighwater reflect the status of SQLite at different points in time -** and it is possible that another thread might change the parameter -** in between the times when *pCurrent and *pHighwater are written. +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. ** ** See also: [sqlite3_db_status()] */ -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int SQLITE_STDCALL sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); /* @@ -6343,6 +6591,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Database Connection Status +** METHOD: sqlite3 ** ** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the @@ -6363,7 +6612,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ -SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* ** CAPI3REF: Status Parameters for database connections @@ -6471,6 +6720,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r /* ** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt ** ** ^(Each prepared statement maintains various ** [SQLITE_STMTSTATUS counters] that measure the number @@ -6492,7 +6742,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ** See also: [sqlite3_status()] and [sqlite3_db_status()]. */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements @@ -6819,6 +7069,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. @@ -6911,20 +7165,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] ** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** -** ^Each call to sqlite3_backup_step() sets two values inside -** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source database file. -** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces -** retrieve these two values, respectively. -** -** ^The values returned by these functions are only updated by -** sqlite3_backup_step(). ^If the source database is modified during a backup -** operation, then the values are not updated to account for any extra -** pages that need to be updated or the size of the source database file -** changing. +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ ** ** Concurrent Usage of Database Handles ** @@ -6957,19 +7211,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( +SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ const char *zDestName, /* Destination database name */ sqlite3 *pSource, /* Source database handle */ const char *zSourceName /* Source database name */ ); -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); /* ** CAPI3REF: Unlock Notification +** METHOD: sqlite3 ** ** ^When running in shared-cache mode, a database operation may fail with ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or @@ -7082,7 +7337,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** the special "DROP TABLE/INDEX" case, the extended error code is just ** SQLITE_LOCKED.)^ */ -SQLITE_API int sqlite3_unlock_notify( +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( sqlite3 *pBlocked, /* Waiting connection */ void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ void *pNotifyArg /* Argument to pass to xNotify */ @@ -7097,8 +7352,8 @@ SQLITE_API int sqlite3_unlock_notify( ** strings in a case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int sqlite3_stricmp(const char *, const char *); -SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *, const char *); +SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing @@ -7113,7 +7368,7 @@ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); ** Note that this routine returns zero on a match and non-zero if the strings ** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. */ -SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); +SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: Error Logging Interface @@ -7136,18 +7391,17 @@ SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); ** a few hundred characters, it will be truncated to the length of the ** buffer. */ -SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); +SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...); /* ** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_hook()] function is used to register a callback that -** will be invoked each time a database connection commits data to a -** [write-ahead log] (i.e. whenever a transaction is committed in -** [journal_mode | journal_mode=WAL mode]). +** is invoked each time data is committed to a database in wal mode. ** -** ^The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked @@ -7173,7 +7427,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will ** those overwrite any prior [sqlite3_wal_hook()] settings. */ -SQLITE_API void *sqlite3_wal_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( sqlite3*, int(*)(void *,sqlite3*,const char*,int), void* @@ -7181,6 +7435,7 @@ SQLITE_API void *sqlite3_wal_hook( /* ** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around ** [sqlite3_wal_hook()] that causes any database on [database connection] D @@ -7207,104 +7462,123 @@ SQLITE_API void *sqlite3_wal_hook( ** is only necessary if the default setting is found to be suboptimal ** for a particular application. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. -** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ -SQLITE_API int sqlite3_wal_checkpoint_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -7313,16 +7587,18 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7338,7 +7614,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options ** may be added in the future. */ -SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); /* ** CAPI3REF: Virtual Table Configuration Options @@ -7391,7 +7667,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** of the SQL statement that triggered the call to the [xUpdate] method of the ** [virtual table]. */ -SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); +SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes @@ -7411,6 +7687,108 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
    +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 + +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** This interface returns information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); /* @@ -7465,7 +7843,7 @@ typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; ** ** SELECT ... FROM WHERE MATCH $zGeom(... params ...) */ -SQLITE_API int sqlite3_rtree_geometry_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), @@ -7491,7 +7869,7 @@ struct sqlite3_rtree_geometry { ** ** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) */ -SQLITE_API int sqlite3_rtree_query_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( sqlite3 *db, const char *zQueryFunc, int (*xQueryFunc)(sqlite3_rtree_query_info*), @@ -7525,6 +7903,8 @@ struct sqlite3_rtree_query_info { int eParentWithin; /* Visibility of parent node */ int eWithin; /* OUT: Visiblity */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ }; /* @@ -7541,3 +7921,523 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. For each row visited, the callback function +** passed as the fourth argument is invoked. The context and API objects +** passed to the callback function may be used to access the properties of +** each matched row. Invoking Api.xUserData() returns a copy of the pointer +** passed as the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iOff>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods. +** +** xPhraseNext() +** See xPhraseFirst above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 1 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and inititalize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
    • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
    • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
    • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
    • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
    +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
    1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
    2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
    3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
    +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + + diff --git a/external/sqlite3/sqlite3ext.h b/external/sqlite3/sqlite3ext.h index f9a066592d..017ea308b1 100644 --- a/external/sqlite3/sqlite3ext.h +++ b/external/sqlite3/sqlite3ext.h @@ -267,6 +267,14 @@ struct sqlite3_api_routines { void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, void(*)(void*), unsigned char); int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); }; /* @@ -280,7 +288,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -407,6 +415,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -497,9 +506,17 @@ struct sqlite3_api_routines { #define sqlite3_result_blob64 sqlite3_api->result_blob64 #define sqlite3_result_text64 sqlite3_api->result_text64 #define sqlite3_strglob sqlite3_api->strglob -#endif /* SQLITE_CORE */ +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; diff --git a/external/sqlite3/src/sqlite3.c b/external/sqlite3/src/sqlite3.c index 9a8a0eac8c..0ae407dd83 100644 --- a/external/sqlite3/src/sqlite3.c +++ b/external/sqlite3/src/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.8.7.2. By combining all the individual C code files into this +** version 3.9.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -22,9 +22,6 @@ #ifndef SQLITE_PRIVATE # define SQLITE_PRIVATE static #endif -#ifndef SQLITE_API -# define SQLITE_API -#endif /************** Begin file sqliteInt.h ***************************************/ /* ** 2001 September 15 @@ -43,6 +40,92 @@ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ +/* +** Include the header file used to customize the compiler options for MSVC. +** This should be done first so that it can successfully prevent spurious +** compiler warnings due to subsequent content in this file and other files +** that are included by this file. +*/ +/************** Include msvc.h in the middle of sqliteInt.h ******************/ +/************** Begin file msvc.h ********************************************/ +/* +** 2015 January 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to MSVC. +*/ +#ifndef _MSVC_H_ +#define _MSVC_H_ + +#if defined(_MSC_VER) +#pragma warning(disable : 4054) +#pragma warning(disable : 4055) +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4130) +#pragma warning(disable : 4152) +#pragma warning(disable : 4189) +#pragma warning(disable : 4206) +#pragma warning(disable : 4210) +#pragma warning(disable : 4232) +#pragma warning(disable : 4244) +#pragma warning(disable : 4305) +#pragma warning(disable : 4306) +#pragma warning(disable : 4702) +#pragma warning(disable : 4706) +#endif /* defined(_MSC_VER) */ + +#endif /* _MSVC_H_ */ + +/************** End of msvc.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Special setup for VxWorks +*/ +/************** Include vxworks.h in the middle of sqliteInt.h ***************/ +/************** Begin file vxworks.h *****************************************/ +/* +** 2015-03-02 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Wind River's VxWorks +*/ +#if defined(__RTP__) || defined(_WRS_KERNEL) +/* This is VxWorks. Set up things specially for that OS +*/ +#include +#include /* amalgamator: dontcache */ +#define OS_VXWORKS 1 +#define SQLITE_OS_OTHER 0 +#define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +#define SQLITE_ENABLE_LOCKING_STYLE 0 +#define HAVE_UTIME 1 +#else +/* This is not VxWorks. */ +#define OS_VXWORKS 0 +#endif /* defined(_WRS_KERNEL) */ + +/************** End of vxworks.h *********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + /* ** These #defines should enable >2GB file support on POSIX if the ** underlying operating system supports it. If the OS lacks @@ -75,6 +158,13 @@ # define _LARGEFILE_SOURCE 1 #endif +/* What version of GCC is being used. 0 means GCC is not being used */ +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif + /* Needed for various definitions... */ #if defined(__GNUC__) && !defined(_GNU_SOURCE) # define _GNU_SOURCE @@ -147,7 +237,7 @@ ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. +** on how SQLite interfaces are supposed to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting @@ -167,21 +257,25 @@ extern "C" { /* -** Add the ability to override 'extern' +** Provide the ability to override linkage features of the interface. */ #ifndef SQLITE_EXTERN # define SQLITE_EXTERN extern #endif - #ifndef SQLITE_API # define SQLITE_API #endif - +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif +#ifndef SQLITE_STDCALL +# define SQLITE_STDCALL +#endif /* ** These no-op macros are used in front of interfaces to mark those ** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards +** should not use deprecated interfaces - they are supported for backwards ** compatibility only. Application writers should be aware that ** experimental interfaces are subject to change in point releases. ** @@ -231,9 +325,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.7.2" -#define SQLITE_VERSION_NUMBER 3008007 -#define SQLITE_SOURCE_ID "2014-11-18 20:57:56 2ab564bf9655b7c7b97ab85cafc8a48329b27f93" +#define SQLITE_VERSION "3.9.2" +#define SQLITE_VERSION_NUMBER 3009002 +#define SQLITE_SOURCE_ID "2015-11-02 18:31:45 bda77dda9697c463c3d0704014d51627fceee328" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -244,7 +338,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** **
    @@ -266,9 +360,9 @@ extern "C" {
     ** See also: [sqlite_version()] and [sqlite_source_id()].
     */
     SQLITE_API const char sqlite3_version[] = SQLITE_VERSION;
    -SQLITE_API const char *sqlite3_libversion(void);
    -SQLITE_API const char *sqlite3_sourceid(void);
    -SQLITE_API int sqlite3_libversion_number(void);
    +SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void);
    +SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void);
    +SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void);
     
     /*
     ** CAPI3REF: Run-Time Library Compilation Options Diagnostics
    @@ -293,8 +387,8 @@ SQLITE_API int sqlite3_libversion_number(void);
     ** [sqlite_compileoption_get()] and the [compile_options pragma].
     */
     #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
    -SQLITE_API int sqlite3_compileoption_used(const char *zOptName);
    -SQLITE_API const char *sqlite3_compileoption_get(int N);
    +SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName);
    +SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N);
     #endif
     
     /*
    @@ -325,7 +419,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
     ** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but
     ** can be fully or partially disabled using a call to [sqlite3_config()]
     ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD],
    -** or [SQLITE_CONFIG_MUTEX].  ^(The return value of the
    +** or [SQLITE_CONFIG_SERIALIZED].  ^(The return value of the
     ** sqlite3_threadsafe() function shows only the compile-time setting of
     ** thread safety, not any run-time changes to that setting made by
     ** sqlite3_config(). In other words, the return value from sqlite3_threadsafe()
    @@ -333,7 +427,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
     **
     ** See the [threading mode] documentation for additional information.
     */
    -SQLITE_API int sqlite3_threadsafe(void);
    +SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void);
     
     /*
     ** CAPI3REF: Database Connection Handle
    @@ -390,6 +484,7 @@ typedef sqlite_uint64 sqlite3_uint64;
     
     /*
     ** CAPI3REF: Closing A Database Connection
    +** DESTRUCTOR: sqlite3
     **
     ** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors
     ** for the [sqlite3] object.
    @@ -429,8 +524,8 @@ typedef sqlite_uint64 sqlite3_uint64;
     ** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer
     ** argument is a harmless no-op.
     */
    -SQLITE_API int sqlite3_close(sqlite3*);
    -SQLITE_API int sqlite3_close_v2(sqlite3*);
    +SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3*);
    +SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3*);
     
     /*
     ** The type for a callback function.
    @@ -441,6 +536,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
     
     /*
     ** CAPI3REF: One-Step Query Execution Interface
    +** METHOD: sqlite3
     **
     ** The sqlite3_exec() interface is a convenience wrapper around
     ** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()],
    @@ -492,7 +588,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
     ** Restrictions:
     **
     ** 
      -**
    • The application must insure that the 1st parameter to sqlite3_exec() +**
    • The application must ensure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. **
    • The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. @@ -500,7 +596,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. **
    */ -SQLITE_API int sqlite3_exec( +SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ @@ -595,6 +691,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) @@ -880,14 +977,16 @@ struct sqlite3_io_methods { ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] ** interface. ** +**
      +**
    • [[SQLITE_FCNTL_LOCKSTATE]] ** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) ** into an integer that the pArg argument points to. This capability -** is used during testing and only needs to be supported when SQLITE_TEST -** is defined. -**
        +** is used during testing and is only available when the SQLITE_TEST +** compile-time option is used. +** **
      • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the @@ -1012,7 +1111,9 @@ struct sqlite3_io_methods { ** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] ** file control returns [SQLITE_OK], then the parser assumes that the ** VFS has handled the PRAGMA itself and the parser generates a no-op -** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** prepared statement if result string is NULL, or that returns a copy +** of the result string if the string is non-NULL. +** ^If the [SQLITE_FCNTL_PRAGMA] file control returns ** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means ** that the VFS encountered an error while handling the [PRAGMA] and the ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] @@ -1070,12 +1171,27 @@ struct sqlite3_io_methods { ** pointed to by the pArg argument. This capability is used during testing ** and only needs to be supported when SQLITE_TEST is defined. ** +**
      • [[SQLITE_FCNTL_WAL_BLOCK]] +** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might +** be advantageous to block on the next WAL lock if the lock is not immediately +** available. The WAL subsystem issues this signal during rare +** circumstances in order to fix a problem with priority inversion. +** Applications should not use this file-control. +** +**
      • [[SQLITE_FCNTL_ZIPVFS]] +** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other +** VFS should return SQLITE_NOTFOUND for this opcode. +** +**
      • [[SQLITE_FCNTL_RBU]] +** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by +** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for +** this opcode. **
      */ #define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 @@ -1094,6 +1210,15 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_WAL_BLOCK 24 +#define SQLITE_FCNTL_ZIPVFS 25 +#define SQLITE_FCNTL_RBU 26 + +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO + /* ** CAPI3REF: Mutex Handle @@ -1345,7 +1470,7 @@ struct sqlite3_vfs { **
    ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED @@ -1442,10 +1567,10 @@ struct sqlite3_vfs { ** must return [SQLITE_OK] on success and some other [error code] upon ** failure. */ -SQLITE_API int sqlite3_initialize(void); -SQLITE_API int sqlite3_shutdown(void); -SQLITE_API int sqlite3_os_init(void); -SQLITE_API int sqlite3_os_end(void); +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void); +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); /* ** CAPI3REF: Configuring The SQLite Library @@ -1456,9 +1581,11 @@ SQLITE_API int sqlite3_os_end(void); ** applications and so this routine is usually not necessary. It is ** provided to support rare applications with unusual needs. ** -** The sqlite3_config() interface is not threadsafe. The application -** must insure that no other SQLite interfaces are invoked by other -** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. +** +** The sqlite3_config() interface ** may only be invoked prior to library initialization using ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before @@ -1476,10 +1603,11 @@ SQLITE_API int sqlite3_os_end(void); ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. */ -SQLITE_API int sqlite3_config(int, ...); +SQLITE_API int SQLITE_CDECL sqlite3_config(int, ...); /* ** CAPI3REF: Configure database connections +** METHOD: sqlite3 ** ** The sqlite3_db_config() interface is used to make configuration ** changes to a [database connection]. The interface is similar to @@ -1494,7 +1622,7 @@ SQLITE_API int sqlite3_config(int, ...); ** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if ** the call is considered successful. */ -SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); /* ** CAPI3REF: Memory Allocation Routines @@ -1628,31 +1756,33 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option. ** ** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The argument specifies +**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies ** alternative low-level memory allocation routines to be used in place of ** the memory allocation routines built into SQLite.)^ ^SQLite makes ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
    ** ** [[SQLITE_CONFIG_GETMALLOC]]
    SQLITE_CONFIG_GETMALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +**
    ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    -**
    ^This option takes single argument of type int, interpreted as a -** boolean, which enables or disables the collection of memory allocation -** statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] **
    • [sqlite3_soft_heap_limit64()] -**
    • [sqlite3_status()] +**
    • [sqlite3_status64()] **
    )^ ** ^Memory allocation statistics are enabled by default unless SQLite is ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory @@ -1660,53 +1790,67 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** scratch memory. There are three arguments: A pointer an 8-byte +**
    ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer +** that SQLite can use for scratch memory. ^(There are three arguments +** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. +** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than one scratch buffers per thread. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.
    +** [sqlite3_malloc()] will be used to obtain the memory needed.

    +** ^When the application provides any amount of scratch memory using +** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large +** [sqlite3_malloc|heap allocations]. +** This can help [Robson proof|prevent memory allocation failures] due to heap +** fragmentation in low-memory embedded systems. +** ** ** [[SQLITE_CONFIG_PAGECACHE]]

    SQLITE_CONFIG_PAGECACHE
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implementation. +**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** that SQLite can use for the database page cache with the default page +** cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. -** There are three arguments to this option: A pointer to 8-byte aligned +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus a little extra for each -** page header. ^The page header size is 20 to 40 bytes depending on -** the host architecture. ^It is harmless, apart from the wasted memory, -** to make sz a little too large. The first -** argument should point to an allocation of at least sz*N bytes of memory. +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The pointer in the first argument must -** be aligned to an 8-byte boundary or subsequent behavior of SQLite -** will be undefined.
    +** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    -**
    ^This option specifies a static memory buffer that SQLite will use -** for all of its dynamic memory allocation needs beyond those provided -** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** There are three arguments: An 8-byte aligned pointer to the memory, +**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the -** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or -** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** memory pointer is not NULL then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. @@ -1714,11 +1858,11 @@ struct sqlite3_mem_methods { ** for the minimum allocation size are 2**5 through 2**8.
    ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The argument specifies -** alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +**
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to @@ -1726,8 +1870,8 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_GETMUTEX]]
    SQLITE_CONFIG_GETMUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The +**
    ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] ** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation @@ -1739,25 +1883,25 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    -**
    ^(This option takes two arguments that determine the default -** memory allocation for the lookaside memory allocator on each -** [database connection]. The first argument is the +**
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** verb to [sqlite3_db_config()] can be used to change the lookaside +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
    ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    -**
    ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods2] object. This object specifies the interface -** to a custom page cache implementation.)^ ^SQLite makes a copy of the -** object and uses it for page cache memory allocations.
    +**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
    ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    -**
    ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
    +**
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite @@ -1780,10 +1924,11 @@ struct sqlite3_mem_methods { ** function must be threadsafe.
    ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI -**
    ^(This option takes a single argument of type int. If non-zero, then -** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled.)^ ^If URI handling is globally enabled, all filenames -** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +**
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1793,9 +1938,10 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined.)^ ** ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN -**
    ^This option takes a single integer argument which is interpreted as -** a boolean in order to enable or disable the use of covering indices for -** full table scans in the query optimizer. ^The default setting is determined +**
    ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" ** if that compile-time option is omitted. ** The ability to disable the use of covering indices for full table scans @@ -1835,18 +1981,37 @@ struct sqlite3_mem_methods { ** ^The default setting can be overridden by each database connection using ** either the [PRAGMA mmap_size] command, or by using the ** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size -** cannot be changed at run-time. Nor may the maximum allowed mmap size -** exceed the compile-time maximum mmap size set by the +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ ** ^If either argument to this option is negative, then that argument is ** changed to its compile-time default. ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
    SQLITE_CONFIG_WIN32_HEAPSIZE -**
    ^This option is only available if SQLite is compiled for Windows -** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +**
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
    SQLITE_CONFIG_PCACHE_HDRSZ +**
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** +** [[SQLITE_CONFIG_PMASZ]] +**
    SQLITE_CONFIG_PMASZ +**
    ^The SQLITE_CONFIG_PMASZ option takes a single parameter which +** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded +** sorter to that integer. The default minimum PMA Size is set by the +** [SQLITE_SORTER_PMASZ] compile-time option. New threads are launched +** to help with sort operations when multithreaded sorting +** is enabled (using the [PRAGMA threads] command) and the amount of content +** to be sorted exceeds the page size times the minimum of the +** [PRAGMA cache_size] setting and this value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1872,6 +2037,8 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ +#define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1938,15 +2105,17 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Enable Or Disable Extended Result Codes +** METHOD: sqlite3 ** ** ^The sqlite3_extended_result_codes() routine enables or disables the ** [extended result codes] feature of SQLite. ^The extended result ** codes are disabled by default for historical compatibility. */ -SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid +** METHOD: sqlite3 ** ** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) ** has a unique 64-bit signed @@ -1994,52 +2163,51 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** unpredictable and might not equal either the old or the new ** last insert [rowid]. */ -SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified +** METHOD: sqlite3 ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. -** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. -** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. +** +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. +** +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
      +**
    • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
    +** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -2048,25 +2216,23 @@ SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified +** METHOD: sqlite3 ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** @@ -2074,10 +2240,11 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); /* ** CAPI3REF: Interrupt A Long-Running Query +** METHOD: sqlite3 ** ** ^This function causes any pending database operation to abort and ** return at its earliest opportunity. This routine is typically @@ -2113,7 +2280,7 @@ SQLITE_API int sqlite3_total_changes(sqlite3*); ** If the database connection closes while [sqlite3_interrupt()] ** is running then bad things will likely happen. */ -SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -2148,11 +2315,13 @@ SQLITE_API void sqlite3_interrupt(sqlite3*); ** The input to [sqlite3_complete16()] must be a zero-terminated ** UTF-16 string in native byte order. */ -SQLITE_API int sqlite3_complete(const char *sql); -SQLITE_API int sqlite3_complete16(const void *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} +** METHOD: sqlite3 ** ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X ** that might be invoked with argument P whenever @@ -2169,7 +2338,7 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for the same locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] is returned ** to the application. @@ -2208,10 +2377,11 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); /* ** CAPI3REF: Set A Busy Timeout +** METHOD: sqlite3 ** ** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps ** for a specified amount of time when a table is locked. ^The handler @@ -2230,10 +2400,11 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** ** See also: [PRAGMA busy_timeout] */ -SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries +** METHOD: sqlite3 ** ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. @@ -2304,7 +2475,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** reflected in subsequent calls to [sqlite3_errcode()] or ** [sqlite3_errmsg()]. */ -SQLITE_API int sqlite3_get_table( +SQLITE_API int SQLITE_STDCALL sqlite3_get_table( sqlite3 *db, /* An open database */ const char *zSql, /* SQL to be evaluated */ char ***pazResult, /* Results of the query */ @@ -2312,13 +2483,17 @@ SQLITE_API int sqlite3_get_table( int *pnColumn, /* Number of result columns written here */ char **pzErrmsg /* Error msg written here */ ); -SQLITE_API void sqlite3_free_table(char **result); +SQLITE_API void SQLITE_STDCALL sqlite3_free_table(char **result); /* ** CAPI3REF: Formatted String Printing Functions ** ** These routines are work-alikes of the "printf()" family of functions ** from the standard C library. +** These routines understand most of the common K&R formatting options, +** plus some additional non-standard formats, detailed below. +** Note that some of the more obscure formatting options from recent +** C-library standards are omitted from this implementation. ** ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their ** results into memory obtained from [sqlite3_malloc()]. @@ -2351,7 +2526,7 @@ SQLITE_API void sqlite3_free_table(char **result); ** These routines all implement some additional formatting ** options that are useful for constructing SQL statements. ** All of the usual printf() formatting options apply. In addition, there -** is are "%q", "%Q", and "%z" options. +** is are "%q", "%Q", "%w" and "%z" options. ** ** ^(The %q option works like %s in that it substitutes a nul-terminated ** string from the argument list. But %q also doubles every '\'' character. @@ -2404,14 +2579,20 @@ SQLITE_API void sqlite3_free_table(char **result); ** The code above will render a correct SQL statement in the zSQL ** variable even if the zText variable is a NULL pointer. ** +** ^(The "%w" formatting option is like "%q" except that it expects to +** be contained within double-quotes instead of single quotes, and it +** escapes the double-quote character instead of the single-quote +** character.)^ The "%w" formatting option is intended for safely inserting +** table and column names into a constructed SQL statement. +** ** ^(The "%z" formatting option works like "%s" but with the ** addition that after the string has been read and copied into ** the result, [sqlite3_free()] is called on the input string.)^ */ -SQLITE_API char *sqlite3_mprintf(const char*,...); -SQLITE_API char *sqlite3_vmprintf(const char*, va_list); -SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); -SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char*,...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list); /* ** CAPI3REF: Memory Allocation Subsystem @@ -2501,12 +2682,12 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** a block of memory after it has been released using ** [sqlite3_free()] or [sqlite3_realloc()]. */ -SQLITE_API void *sqlite3_malloc(int); -SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); -SQLITE_API void *sqlite3_realloc(void*, int); -SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); -SQLITE_API void sqlite3_free(void*); -SQLITE_API sqlite3_uint64 sqlite3_msize(void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void*, int); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void SQLITE_STDCALL sqlite3_free(void*); +SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2531,8 +2712,8 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** by [sqlite3_memory_highwater(1)] is the high-water mark ** prior to the reset. */ -SQLITE_API sqlite3_int64 sqlite3_memory_used(void); -SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag); /* ** CAPI3REF: Pseudo-Random Number Generator @@ -2544,20 +2725,22 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ -SQLITE_API void sqlite3_randomness(int N, void *P); +SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P); /* ** CAPI3REF: Compile-Time Authorization Callbacks +** METHOD: sqlite3 ** ** ^This routine registers an authorizer callback with a particular ** [database connection], supplied in the first argument. @@ -2636,7 +2819,7 @@ SQLITE_API void sqlite3_randomness(int N, void *P); ** as stated in the previous paragraph, sqlite3_step() invokes ** sqlite3_prepare_v2() to reprepare a statement after a schema change. */ -SQLITE_API int sqlite3_set_authorizer( +SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3*, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pUserData @@ -2714,6 +2897,7 @@ SQLITE_API int sqlite3_set_authorizer( /* ** CAPI3REF: Tracing And Profiling Functions +** METHOD: sqlite3 ** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. @@ -2740,12 +2924,13 @@ SQLITE_API int sqlite3_set_authorizer( ** sqlite3_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ -SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); -SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); /* ** CAPI3REF: Query Progress Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to @@ -2775,10 +2960,11 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, ** database connections for the meaning of "modify" in this paragraph. ** */ -SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection +** CONSTRUCTOR: sqlite3 ** ** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for @@ -3003,15 +3189,15 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** See also: [sqlite3_temp_directory] */ -SQLITE_API int sqlite3_open( +SQLITE_API int SQLITE_STDCALL sqlite3_open( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open16( +SQLITE_API int SQLITE_STDCALL sqlite3_open16( const void *filename, /* Database filename (UTF-16) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb, /* OUT: SQLite db handle */ int flags, /* Flags */ @@ -3057,19 +3243,22 @@ SQLITE_API int sqlite3_open_v2( ** VFS method, then the behavior of this routine is undefined and probably ** undesirable. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* ** CAPI3REF: Error Codes And Messages -** -** ^The sqlite3_errcode() interface returns the numeric [result code] or -** [extended result code] for the most recent failed sqlite3_* API call -** associated with a [database connection]. If a prior API call failed -** but the most recent API call succeeded, the return value from -** sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() +** METHOD: sqlite3 +** +** ^If the most recent sqlite3_* API call associated with +** [database connection] D failed, then the sqlite3_errcode(D) interface +** returns the numeric [result code] or [extended result code] for that +** API call. +** If the most recent API call was successful, +** then the return value from sqlite3_errcode() is undefined. +** ^The sqlite3_extended_errcode() ** interface is the same except that it always returns the ** [extended result code] even when extended result codes are ** disabled. @@ -3100,40 +3289,41 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** was invoked incorrectly by the application. In that case, the ** error code and message may or may not be set. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db); -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); -SQLITE_API const char *sqlite3_errmsg(sqlite3*); -SQLITE_API const void *sqlite3_errmsg16(sqlite3*); -SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int); /* -** CAPI3REF: SQL Statement Object +** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} ** -** An instance of this object represents a single SQL statement. -** This object is variously known as a "prepared statement" or a -** "compiled SQL statement" or simply as a "statement". +** An instance of this object represents a single SQL statement that +** has been compiled into binary form and is ready to be evaluated. ** -** The life of a statement object goes something like this: +** Think of each SQL statement as a separate computer program. The +** original SQL text is source code. A prepared statement object +** is the compiled object code. All SQL must be converted into a +** prepared statement before it can be run. +** +** The life-cycle of a prepared statement object usually goes like this: ** **
      -**
    1. Create the object using [sqlite3_prepare_v2()] or a related -** function. -**
    2. Bind values to [host parameters] using the sqlite3_bind_*() +**
    3. Create the prepared statement object using [sqlite3_prepare_v2()]. +**
    4. Bind values to [parameters] using the sqlite3_bind_*() ** interfaces. **
    5. Run the SQL by calling [sqlite3_step()] one or more times. -**
    6. Reset the statement using [sqlite3_reset()] then go back +**
    7. Reset the prepared statement using [sqlite3_reset()] then go back ** to step 2. Do this zero or more times. **
    8. Destroy the object using [sqlite3_finalize()]. **
    -** -** Refer to documentation on individual methods above for additional -** information. */ typedef struct sqlite3_stmt sqlite3_stmt; /* ** CAPI3REF: Run-time Limits +** METHOD: sqlite3 ** ** ^(This interface allows the size of various constructs to be limited ** on a connection by connection basis. The first parameter is the @@ -3171,7 +3361,7 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** ** New run-time limit categories may be added in future releases. */ -SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Run-Time Limit Categories @@ -3245,6 +3435,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Compiling An SQL Statement ** KEYWORDS: {SQL statement compiler} +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_stmt ** ** To execute an SQL query, it must first be compiled into a byte-code ** program using one of these routines. @@ -3258,16 +3450,14 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() ** use UTF-16. ** -** ^If the nByte argument is less than zero, then zSql is read up to the -** first zero terminator. ^If nByte is non-negative, then it is the maximum -** number of bytes read from zSql. ^When nByte is non-negative, the -** zSql string ends at either the first '\000' or '\u0000' character or -** the nByte-th byte, whichever comes first. If the caller knows -** that the supplied string is nul-terminated, then there is a small -** performance advantage to be gained by passing an nByte parameter that -** is equal to the number of bytes in the input string including -** the nul-terminator bytes as this saves SQLite from having to -** make a copy of the input string. +** ^If the nByte argument is negative, then zSql is read up to the +** first zero terminator. ^If nByte is positive, then it is the +** number of bytes read from zSql. ^If nByte is zero, then no prepared +** statement is generated. +** If the caller knows that the supplied string is nul-terminated, then +** there is a small performance advantage to passing an nByte parameter that +** is the number of bytes in the input string including +** the nul-terminator. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -3323,28 +3513,28 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** */ -SQLITE_API int sqlite3_prepare( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const void **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ @@ -3354,15 +3544,17 @@ SQLITE_API int sqlite3_prepare16_v2( /* ** CAPI3REF: Retrieving Statement SQL +** METHOD: sqlite3_stmt ** ** ^This interface can be used to retrieve a saved copy of the original ** SQL text used to create a [prepared statement] if that statement was ** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. */ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database +** METHOD: sqlite3_stmt ** ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if ** and only if the [prepared statement] X makes no direct changes to @@ -3390,14 +3582,16 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** change the configuration of a database connection, they do not make ** changes to the content of the database files on disk. */ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** METHOD: sqlite3_stmt ** ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the ** [prepared statement] S has been stepped at least once using -** [sqlite3_step(S)] but has not run to completion and/or has not +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor ** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) ** interface returns false if S is a NULL pointer. If S is not a ** NULL pointer and is not a pointer to a valid [prepared statement] @@ -3409,7 +3603,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** for example, in diagnostic routines to search for prepared ** statements that are holding a transaction open. */ -SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*); /* ** CAPI3REF: Dynamically Typed Value Object @@ -3424,7 +3618,9 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); ** Some interfaces require a protected sqlite3_value. Other interfaces ** will accept either a protected or an unprotected sqlite3_value. ** Every interface that accepts sqlite3_value arguments specifies -** whether or not it requires a protected sqlite3_value. +** whether or not it requires a protected sqlite3_value. The +** [sqlite3_value_dup()] interface can be used to construct a new +** protected sqlite3_value from an unprotected sqlite3_value. ** ** The terms "protected" and "unprotected" refer to whether or not ** a mutex is held. An internal mutex is held for a protected @@ -3468,6 +3664,7 @@ typedef struct sqlite3_context sqlite3_context; ** CAPI3REF: Binding Values To Prepared Statements ** KEYWORDS: {host parameter} {host parameters} {host parameter name} ** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, ** literals may be replaced by a [parameter] that matches one of following @@ -3570,22 +3767,24 @@ typedef struct sqlite3_context sqlite3_context; ** See also: [sqlite3_bind_parameter_count()], ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); -SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, void(*)(void*)); -SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); -SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); -SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); -SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); -SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); -SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); /* ** CAPI3REF: Number Of SQL Parameters +** METHOD: sqlite3_stmt ** ** ^This routine can be used to find the number of [SQL parameters] ** in a [prepared statement]. SQL parameters are tokens of the @@ -3602,10 +3801,11 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); ** [sqlite3_bind_parameter_name()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt*); /* ** CAPI3REF: Name Of A Host Parameter +** METHOD: sqlite3_stmt ** ** ^The sqlite3_bind_parameter_name(P,N) interface returns ** the name of the N-th [SQL parameter] in the [prepared statement] P. @@ -3629,10 +3829,11 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); ** [sqlite3_bind_parameter_count()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*, int); /* ** CAPI3REF: Index Of A Parameter With A Given Name +** METHOD: sqlite3_stmt ** ** ^Return the index of an SQL parameter given its name. ^The ** index value returned is suitable for use as the second @@ -3643,21 +3844,23 @@ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); ** ** See also: [sqlite3_bind_blob|sqlite3_bind()], ** [sqlite3_bind_parameter_count()], and -** [sqlite3_bind_parameter_index()]. +** [sqlite3_bind_parameter_name()]. */ -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); /* ** CAPI3REF: Reset All Bindings On A Prepared Statement +** METHOD: sqlite3_stmt ** ** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset ** the [sqlite3_bind_blob | bindings] on a [prepared statement]. ** ^Use this routine to reset all host parameters to NULL. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt*); /* ** CAPI3REF: Number Of Columns In A Result Set +** METHOD: sqlite3_stmt ** ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL @@ -3665,10 +3868,11 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ** See also: [sqlite3_data_count()] */ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set +** METHOD: sqlite3_stmt ** ** ^These routines return the name assigned to a particular column ** in the result set of a [SELECT] statement. ^The sqlite3_column_name() @@ -3693,11 +3897,12 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); ** then the name of the column is unspecified and may change from ** one release of SQLite to the next. */ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N); /* ** CAPI3REF: Source Of Data In A Query Result +** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and ** table column that is the origin of a particular result column in @@ -3741,15 +3946,16 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** for the same [prepared statement] and result column ** at the same time then the results are undefined. */ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt*,int); /* ** CAPI3REF: Declared Datatype Of A Query Result +** METHOD: sqlite3_stmt ** ** ^(The first parameter is a [prepared statement]. ** If this statement is a [SELECT] statement and the Nth column of the @@ -3777,11 +3983,12 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); ** is associated with individual values, not with the containers ** used to hold those values. */ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,int); /* ** CAPI3REF: Evaluate An SQL Statement +** METHOD: sqlite3_stmt ** ** After a [prepared statement] has been prepared using either ** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy @@ -3857,10 +4064,11 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** then the more specific [error codes] are returned directly ** by sqlite3_step(). The use of the "v2" interface is recommended. */ -SQLITE_API int sqlite3_step(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set +** METHOD: sqlite3_stmt ** ** ^The sqlite3_data_count(P) interface returns the number of columns in the ** current row of the result set of [prepared statement] P. @@ -3877,7 +4085,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** ** See also: [sqlite3_column_count()] */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes @@ -3914,8 +4122,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Result Values From A Query ** KEYWORDS: {column access functions} -** -** These routines form the "result set" interface. +** METHOD: sqlite3_stmt ** ** ^These routines return information about a single column of the current ** result row of a query. ^In every case the first argument is a pointer @@ -3976,13 +4183,14 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** -** ^The object returned by [sqlite3_column_value()] is an -** [unprotected sqlite3_value] object. An unprotected sqlite3_value object -** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by ** [sqlite3_column_value()] is used in any other way, including calls ** to routines like [sqlite3_value_int()], [sqlite3_value_text()], -** or [sqlite3_value_bytes()], then the behavior is undefined. +** or [sqlite3_value_bytes()], the behavior is not threadsafe. ** ** These routines attempt to convert the value where appropriate. ^For ** example, if the internal representation is FLOAT and a text result @@ -4013,12 +4221,6 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ** )^ ** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** own equivalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. -** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. @@ -4043,7 +4245,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** **
      @@ -4063,7 +4265,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** @@ -4073,19 +4275,20 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** pointer. Subsequent calls to [sqlite3_errcode()] will return ** [SQLITE_NOMEM].)^ */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); -SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors @@ -4109,10 +4312,11 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** statement after it has been finalized can result in undefined and ** undesirable behavior such as segfaults and heap corruption. */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt ** ** The sqlite3_reset() function is called to reset a [prepared statement] ** object back to its initial state, ready to be re-executed. @@ -4135,13 +4339,14 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} +** METHOD: sqlite3 ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior @@ -4234,7 +4439,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. */ -SQLITE_API int sqlite3_create_function( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4244,7 +4449,7 @@ SQLITE_API int sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( sqlite3 *db, const void *zFunctionName, int nArg, @@ -4254,7 +4459,7 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4272,9 +4477,9 @@ SQLITE_API int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ @@ -4296,25 +4501,26 @@ SQLITE_API int sqlite3_create_function_v2( ** These functions are [deprecated]. In order to maintain ** backwards compatibility with older code, these functions continue ** to be supported. However, new applications should avoid -** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you what they do. +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. */ #ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); -SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), void*,sqlite3_int64); #endif /* -** CAPI3REF: Obtaining SQL Function Parameter Values +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on -** the function or aggregate. +** the function or aggregate. ** ** The xFunc (for scalar functions) or xStep (for aggregates) parameters ** to [sqlite3_create_function()] and [sqlite3_create_function16()] @@ -4354,21 +4560,55 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); -SQLITE_API double sqlite3_value_double(sqlite3_value*); -SQLITE_API int sqlite3_value_int(sqlite3_value*); -SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); -SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); -SQLITE_API int sqlite3_value_type(sqlite3_value*); -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value*); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); + +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** SQLite makes no use of subtype itself. It merely passes the subtype +** from the result of one [application-defined SQL function] into the +** input of another. +*/ +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API SQLITE_EXPERIMENTAL sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value*); +SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context ** ** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. @@ -4409,10 +4649,11 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** This routine must be called from the same thread in which ** the aggregate SQL function is running. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); +SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_user_data() interface returns a copy of ** the pointer that was the pUserData parameter (the 5th parameter) @@ -4423,10 +4664,11 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); ** This routine must be called from the same thread in which ** the application-defined function is running. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context*); +SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context*); /* ** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context ** ** ^The sqlite3_context_db_handle() interface returns a copy of ** the pointer to the [database connection] (the 1st parameter) @@ -4434,10 +4676,11 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context*); ** and [sqlite3_create_function16()] routines that originally ** registered the application defined function. */ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to ** associate metadata with argument values. If the same value is passed to @@ -4486,8 +4729,8 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** These routines must be called from the same thread in which ** the SQL function is running. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); -SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); /* @@ -4510,6 +4753,7 @@ typedef void (*sqlite3_destructor_type)(void*); /* ** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See @@ -4525,9 +4769,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite3_result_zeroblob() interfaces set the result of -** the application-defined function to be a BLOB containing all zero -** bytes and N bytes in size, where N is the value of the 2nd parameter. +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. ** ** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified @@ -4609,7 +4853,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** from [sqlite3_malloc()] before it returns. ** ** ^The sqlite3_result_value() interface sets the result of -** the application-defined function to be a copy the +** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The ** sqlite3_result_value() interface makes a copy of the [sqlite3_value] ** so that the [sqlite3_value] specified in the parameter may change or @@ -4622,28 +4866,46 @@ typedef void (*sqlite3_destructor_type)(void*); ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ -SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); -SQLITE_API void sqlite3_result_double(sqlite3_context*, double); -SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); -SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); -SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); -SQLITE_API void sqlite3_result_null(sqlite3_context*); -SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned int); /* ** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 ** ** ^These functions add, remove, or modify a [collation] associated ** with the [database connection] specified as the first argument. @@ -4721,14 +4983,14 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ -SQLITE_API int sqlite3_create_collation( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); -SQLITE_API int sqlite3_create_collation_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, @@ -4736,7 +4998,7 @@ SQLITE_API int sqlite3_create_collation_v2( int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); -SQLITE_API int sqlite3_create_collation16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, @@ -4746,6 +5008,7 @@ SQLITE_API int sqlite3_create_collation16( /* ** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 ** ** ^To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the @@ -4770,12 +5033,12 @@ SQLITE_API int sqlite3_create_collation16( ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. */ -SQLITE_API int sqlite3_collation_needed( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const char*) ); -SQLITE_API int sqlite3_collation_needed16( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const void*) @@ -4789,11 +5052,11 @@ SQLITE_API int sqlite3_collation_needed16( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_key( +SQLITE_API int SQLITE_STDCALL sqlite3_key( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The key */ ); -SQLITE_API int sqlite3_key_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_key_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The key */ @@ -4807,11 +5070,11 @@ SQLITE_API int sqlite3_key_v2( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_rekey( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The new key */ ); -SQLITE_API int sqlite3_rekey_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The new key */ @@ -4821,7 +5084,7 @@ SQLITE_API int sqlite3_rekey_v2( ** Specify the activation key for a SEE database. Unless ** activated, none of the SEE routines will work. */ -SQLITE_API void sqlite3_activate_see( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_see( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -4831,7 +5094,7 @@ SQLITE_API void sqlite3_activate_see( ** Specify the activation key for a CEROD database. Unless ** activated, none of the CEROD routines will work. */ -SQLITE_API void sqlite3_activate_cerod( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_cerod( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -4853,7 +5116,7 @@ SQLITE_API void sqlite3_activate_cerod( ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. */ -SQLITE_API int sqlite3_sleep(int); +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -4953,6 +5216,7 @@ SQLITE_API char *sqlite3_data_directory; /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 ** ** ^The sqlite3_get_autocommit() interface returns non-zero or ** zero if the given database connection is or is not in autocommit mode, @@ -4971,10 +5235,11 @@ SQLITE_API char *sqlite3_data_directory; ** connection while this routine is running, then the return value ** is undefined. */ -SQLITE_API int sqlite3_get_autocommit(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3*); /* ** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt ** ** ^The sqlite3_db_handle interface returns the [database connection] handle ** to which a [prepared statement] belongs. ^The [database connection] @@ -4983,10 +5248,11 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); ** to the [sqlite3_prepare_v2()] call (or its variants) that was used to ** create the statement in the first place. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt*); /* ** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename ** associated with database N of connection D. ^The main database file @@ -4999,19 +5265,21 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** will be an absolute pathname, even if the filename used ** to open the database originally was a URI or relative pathname. */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 ** ** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N ** of connection D is read-only, 0 if it is read/write, or -1 if N is not ** the name of a database on connection D. */ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); +SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 ** ** ^This interface returns a pointer to the next [prepared statement] after ** pStmt associated with the [database connection] pDb. ^If pStmt is NULL @@ -5023,10 +5291,11 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); ** [sqlite3_next_stmt(D,S)] must refer to an open database ** connection and in particular must not be a NULL pointer. */ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); +SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_commit_hook() interface registers a callback ** function to be invoked whenever a transaction is [COMMIT | committed]. @@ -5071,11 +5340,12 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** ** See also the [sqlite3_update_hook()] interface. */ -SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); -SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* ** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 ** ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument @@ -5122,7 +5392,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] ** interfaces. */ -SQLITE_API void *sqlite3_update_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( sqlite3*, void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* @@ -5152,12 +5422,17 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. +** ** This interface is threadsafe on processors where writing a ** 32-bit integer is atomic. ** ** See Also: [SQLite Shared-Cache Mode] */ -SQLITE_API int sqlite3_enable_shared_cache(int); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int); /* ** CAPI3REF: Attempt To Free Heap Memory @@ -5173,10 +5448,11 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** ** See also: [sqlite3_db_release_memory()] */ -SQLITE_API int sqlite3_release_memory(int); +SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int); /* ** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 ** ** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap ** memory as possible from database connection D. Unlike the @@ -5186,7 +5462,7 @@ SQLITE_API int sqlite3_release_memory(int); ** ** See also: [sqlite3_release_memory()] */ -SQLITE_API int sqlite3_db_release_memory(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); /* ** CAPI3REF: Impose A Limit On Heap Size @@ -5238,7 +5514,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** The circumstances under which SQLite will enforce the soft heap limit may ** changes in future releases of SQLite. */ -SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 N); /* ** CAPI3REF: Deprecated Soft Heap Limit Interface @@ -5249,26 +5525,34 @@ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); ** only. All new applications should use the ** [sqlite3_soft_heap_limit64()] interface rather than this one. */ -SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table -** -** ^This routine returns metadata about a specific column of a specific -** database table accessible using the [database connection] handle -** passed as the first function argument. +** METHOD: sqlite3 +** +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR and if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existance of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. ** ** ^The column is identified by the second, third and fourth parameters to -** this function. ^The second parameter is either the name of the database +** this function. ^(The second parameter is either the name of the database ** (i.e. "main", "temp", or an attached database) containing the specified -** table or NULL. ^If it is NULL, then all attached databases are searched +** table or NULL.)^ ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** ** ^The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** name of the desired column, respectively. ** ** ^Metadata is returned by writing to the memory locations passed as the 5th ** and subsequent parameters to this function. ^Any of these arguments may be @@ -5287,16 +5571,17 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** )^ ** ** ^The memory pointed to by the character pointers returned for the -** declaration type and collation sequence is valid only until the next +** declaration type and collation sequence is valid until the next ** call to any SQLite API function. ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no -** explicitly declared [INTEGER PRIMARY KEY] column, then the output -** parameters are set as follows: +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: ** **
       **     data type: "INTEGER"
      @@ -5306,15 +5591,11 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
       **     auto increment: 0
       ** 
      )^ ** -** ^(This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ -** -** ^This API is only available if the library was compiled with the -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ -SQLITE_API int sqlite3_table_column_metadata( +SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -5328,6 +5609,7 @@ SQLITE_API int sqlite3_table_column_metadata( /* ** CAPI3REF: Load An Extension +** METHOD: sqlite3 ** ** ^This interface loads an SQLite extension library from the named file. ** @@ -5360,7 +5642,7 @@ SQLITE_API int sqlite3_table_column_metadata( ** ** See also the [load_extension() SQL function]. */ -SQLITE_API int sqlite3_load_extension( +SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Derived from zFile if 0 */ @@ -5369,6 +5651,7 @@ SQLITE_API int sqlite3_load_extension( /* ** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 ** ** ^So as not to open security holes in older applications that are ** unprepared to deal with [extension loading], and as a means of disabling @@ -5380,7 +5663,7 @@ SQLITE_API int sqlite3_load_extension( ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* ** CAPI3REF: Automatically Load Statically Linked Extensions @@ -5418,7 +5701,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** See also: [sqlite3_reset_auto_extension()] ** and [sqlite3_cancel_auto_extension()] */ -SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); +SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Cancel Automatic Extension Loading @@ -5430,7 +5713,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); ** unregistered and it returns 0 if X was not on the list of initialization ** routines. */ -SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); +SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading @@ -5438,7 +5721,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); ** ^This interface disables all automatic extensions previously ** registered using [sqlite3_auto_extension()]. */ -SQLITE_API void sqlite3_reset_auto_extension(void); +SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void); /* ** The interface to the virtual-table mechanism is currently considered @@ -5565,13 +5848,31 @@ struct sqlite3_module { ** ^The estimatedRows value is an estimate of the number of rows that ** will be returned by the strategy. ** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info ** structure for SQLite version 3.8.2. If a virtual table extension is ** used with an SQLite version earlier than 3.8.2, the results of attempting ** to read or write the estimatedRows field are undefined (but are likely ** to included crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a -** value greater than or equal to 3008002. +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for version 3.9.0. It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. */ struct sqlite3_index_info { /* Inputs */ @@ -5599,8 +5900,15 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ /* Fields below are only available in SQLite 3.8.2 and later */ sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ }; +/* +** CAPI3REF: Virtual Table Scan Flags +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** @@ -5618,6 +5926,7 @@ struct sqlite3_index_info { /* ** CAPI3REF: Register A Virtual Table Implementation +** METHOD: sqlite3 ** ** ^These routines are used to register a new [virtual table module] name. ** ^Module names must be registered before @@ -5641,13 +5950,13 @@ struct sqlite3_index_info { ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ -SQLITE_API int sqlite3_create_module( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ const sqlite3_module *p, /* Methods for the module */ void *pClientData /* Client data for xCreate/xConnect */ ); -SQLITE_API int sqlite3_create_module_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ const sqlite3_module *p, /* Methods for the module */ @@ -5675,7 +5984,7 @@ SQLITE_API int sqlite3_create_module_v2( */ struct sqlite3_vtab { const sqlite3_module *pModule; /* The module for this virtual table */ - int nRef; /* NO LONGER USED */ + int nRef; /* Number of open cursors */ char *zErrMsg; /* Error message from sqlite3_mprintf() */ /* Virtual table implementations will typically add additional fields */ }; @@ -5710,10 +6019,11 @@ struct sqlite3_vtab_cursor { ** to declare the format (the names and datatypes of the columns) of ** the virtual tables they implement. */ -SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); +SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3*, const char *zSQL); /* ** CAPI3REF: Overload A Function For A Virtual Table +** METHOD: sqlite3 ** ** ^(Virtual tables can provide alternative implementations of functions ** using the [xFindFunction] method of the [virtual table module]. @@ -5728,7 +6038,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); ** purpose is to be a placeholder function that can be overloaded ** by a [virtual table]. */ -SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); /* ** The interface to the virtual-table mechanism defined above (back up @@ -5756,6 +6066,8 @@ typedef struct sqlite3_blob sqlite3_blob; /* ** CAPI3REF: Open A BLOB For Incremental I/O +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_blob ** ** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located ** in row iRow, column zColumn, table zTable in database zDb; @@ -5765,26 +6077,42 @@ typedef struct sqlite3_blob sqlite3_blob; ** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; ** )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. -** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". -** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
        +**
      • ^(Database zDb does not exist)^, +**
      • ^(Table zTable does not exist within database zDb)^, +**
      • ^(Table zTable is a WITHOUT ROWID table)^, +**
      • ^(Column zColumn does not exist)^, +**
      • ^(Row iRow is not present in the table)^, +**
      • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
      • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
      • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
      +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5802,18 +6130,14 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. */ -SQLITE_API int sqlite3_blob_open( +SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( sqlite3*, const char *zDb, const char *zTable, @@ -5825,6 +6149,7 @@ SQLITE_API int sqlite3_blob_open( /* ** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob ** ** ^This function is used to move an existing blob handle so that it points ** to a different row of the same database table. ^The new row is identified @@ -5845,34 +6170,34 @@ SQLITE_API int sqlite3_blob_open( ** ** ^This function sets the database handle error code and message. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob ** -** ^Closes an open [BLOB handle]. -** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. -** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ -SQLITE_API int sqlite3_blob_close(sqlite3_blob *); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *); /* ** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The @@ -5884,10 +6209,11 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *); ** been closed by [sqlite3_blob_close()]. Passing any other pointer in ** to this routine results in undefined and probably undesirable behavior. */ -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *); /* ** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob ** ** ^(This function is used to read data from an open [BLOB handle] into a ** caller-supplied buffer. N bytes of data are copied into buffer Z @@ -5912,26 +6238,33 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); ** ** See also: [sqlite3_blob_write()]. */ -SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5940,9 +6273,6 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in @@ -5950,7 +6280,7 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** ** See also: [sqlite3_blob_read()]. */ -SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); +SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); /* ** CAPI3REF: Virtual File System Objects @@ -5981,9 +6311,9 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOff ** ^(If the default VFS is unregistered, another VFS is chosen as ** the default. The choice for the new VFS is arbitrary.)^ */ -SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); -SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); -SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); +SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); /* ** CAPI3REF: Mutexes @@ -5995,34 +6325,34 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
        **
      • SQLITE_MUTEX_PTHREADS **
      • SQLITE_MUTEX_W32 **
      • SQLITE_MUTEX_NOOP -**
      )^ +**
    ** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
      **
    • SQLITE_MUTEX_FAST @@ -6035,7 +6365,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); **
    • SQLITE_MUTEX_STATIC_PMEM **
    • SQLITE_MUTEX_STATIC_APP1 **
    • SQLITE_MUTEX_STATIC_APP2 -**
    )^ +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -6043,14 +6377,14 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -6059,16 +6393,13 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -6076,23 +6407,21 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -6100,11 +6429,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ -SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); -SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex*); /* ** CAPI3REF: Mutex Methods Object @@ -6113,9 +6442,9 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -6156,13 +6485,13 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6188,34 +6517,34 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG -SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); #endif /* @@ -6241,9 +6570,13 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ /* ** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 ** ** ^This interface returns a pointer the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument @@ -6251,10 +6584,11 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); ** ^If the [threading mode] is Single-thread or Multi-thread then this ** routine returns a NULL pointer. */ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -6285,7 +6619,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** ** See also: [SQLITE_FCNTL_LOCKSTATE] */ -SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); /* ** CAPI3REF: Testing Interface @@ -6304,7 +6638,7 @@ SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void* ** Unlike most of the SQLite API, this function is not guaranteed to ** operate consistently from one release to the next. */ -SQLITE_API int sqlite3_test_control(int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); /* ** CAPI3REF: Testing Interface Operation Codes @@ -6338,12 +6672,13 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_BYTEORDER 22 #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 -#define SQLITE_TESTCTRL_LAST 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_LAST 25 /* ** CAPI3REF: SQLite Runtime Status ** -** ^This interface is used to retrieve runtime status information +** ^These interfaces are used to retrieve runtime status information ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes @@ -6357,19 +6692,22 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_status() routine returns SQLITE_OK on success and a -** non-zero [error code] on failure. +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. ** -** This routine is threadsafe but is not atomic. This routine can be -** called while other threads are running the same or different SQLite -** interfaces. However the values returned in *pCurrent and -** *pHighwater reflect the status of SQLite at different points in time -** and it is possible that another thread might change the parameter -** in between the times when *pCurrent and *pHighwater are written. +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. ** ** See also: [sqlite3_db_status()] */ -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int SQLITE_STDCALL sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); /* @@ -6467,6 +6805,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Database Connection Status +** METHOD: sqlite3 ** ** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the @@ -6487,7 +6826,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ -SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* ** CAPI3REF: Status Parameters for database connections @@ -6595,6 +6934,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r /* ** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt ** ** ^(Each prepared statement maintains various ** [SQLITE_STMTSTATUS counters] that measure the number @@ -6616,7 +6956,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ** See also: [sqlite3_status()] and [sqlite3_db_status()]. */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements @@ -6943,6 +7283,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. @@ -7035,20 +7379,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] ** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** -** ^Each call to sqlite3_backup_step() sets two values inside -** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source database file. -** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces -** retrieve these two values, respectively. -** -** ^The values returned by these functions are only updated by -** sqlite3_backup_step(). ^If the source database is modified during a backup -** operation, then the values are not updated to account for any extra -** pages that need to be updated or the size of the source database file -** changing. +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ ** ** Concurrent Usage of Database Handles ** @@ -7081,19 +7425,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( +SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ const char *zDestName, /* Destination database name */ sqlite3 *pSource, /* Source database handle */ const char *zSourceName /* Source database name */ ); -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); /* ** CAPI3REF: Unlock Notification +** METHOD: sqlite3 ** ** ^When running in shared-cache mode, a database operation may fail with ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or @@ -7206,7 +7551,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** the special "DROP TABLE/INDEX" case, the extended error code is just ** SQLITE_LOCKED.)^ */ -SQLITE_API int sqlite3_unlock_notify( +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( sqlite3 *pBlocked, /* Waiting connection */ void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ void *pNotifyArg /* Argument to pass to xNotify */ @@ -7221,8 +7566,8 @@ SQLITE_API int sqlite3_unlock_notify( ** strings in a case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int sqlite3_stricmp(const char *, const char *); -SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *, const char *); +SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing @@ -7237,7 +7582,7 @@ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); ** Note that this routine returns zero on a match and non-zero if the strings ** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. */ -SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); +SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: Error Logging Interface @@ -7260,18 +7605,17 @@ SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); ** a few hundred characters, it will be truncated to the length of the ** buffer. */ -SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); +SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...); /* ** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_hook()] function is used to register a callback that -** will be invoked each time a database connection commits data to a -** [write-ahead log] (i.e. whenever a transaction is committed in -** [journal_mode | journal_mode=WAL mode]). +** is invoked each time data is committed to a database in wal mode. ** -** ^The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked @@ -7297,7 +7641,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will ** those overwrite any prior [sqlite3_wal_hook()] settings. */ -SQLITE_API void *sqlite3_wal_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( sqlite3*, int(*)(void *,sqlite3*,const char*,int), void* @@ -7305,6 +7649,7 @@ SQLITE_API void *sqlite3_wal_hook( /* ** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 ** ** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around ** [sqlite3_wal_hook()] that causes any database on [database connection] D @@ -7331,104 +7676,123 @@ SQLITE_API void *sqlite3_wal_hook( ** is only necessary if the default setting is found to be suboptimal ** for a particular application. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. -** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ -SQLITE_API int sqlite3_wal_checkpoint_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -7437,16 +7801,18 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7462,7 +7828,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options ** may be added in the future. */ -SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); /* ** CAPI3REF: Virtual Table Configuration Options @@ -7515,7 +7881,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** of the SQL statement that triggered the call to the [xUpdate] method of the ** [virtual table]. */ -SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); +SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes @@ -7535,6 +7901,108 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
    +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 + +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** This interface returns information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); /* @@ -7589,7 +8057,7 @@ typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; ** ** SELECT ... FROM WHERE MATCH $zGeom(... params ...) */ -SQLITE_API int sqlite3_rtree_geometry_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), @@ -7615,7 +8083,7 @@ struct sqlite3_rtree_geometry { ** ** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) */ -SQLITE_API int sqlite3_rtree_query_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( sqlite3 *db, const char *zQueryFunc, int (*xQueryFunc)(sqlite3_rtree_query_info*), @@ -7649,6 +8117,8 @@ struct sqlite3_rtree_query_info { int eParentWithin; /* Visibility of parent node */ int eWithin; /* OUT: Visiblity */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ }; /* @@ -7665,6 +8135,526 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. For each row visited, the callback function +** passed as the fourth argument is invoked. The context and API objects +** passed to the callback function may be used to access the properties of +** each matched row. Invoking Api.xUserData() returns a copy of the pointer +** passed as the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iOff>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods. +** +** xPhraseNext() +** See xPhraseFirst above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 1 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and inititalize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
    • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
    • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
    • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
    • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
    +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
    1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
    2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
    3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
    +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -7779,15 +8769,17 @@ struct sqlite3_rtree_query_info { #endif /* -** The maximum number of in-memory pages to use for the main database -** table and for temporary tables. The SQLITE_DEFAULT_CACHE_SIZE +** The suggested maximum number of in-memory pages to use for +** the main database table and for temporary tables. +** +** IMPLEMENTATION-OF: R-31093-59126 The default suggested cache size +** is 2000 pages. +** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be +** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options. */ #ifndef SQLITE_DEFAULT_CACHE_SIZE # define SQLITE_DEFAULT_CACHE_SIZE 2000 #endif -#ifndef SQLITE_DEFAULT_TEMP_CACHE_SIZE -# define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 -#endif /* ** The default number of frames to accumulate in the log file before @@ -7952,6 +8944,24 @@ struct sqlite3_rtree_query_info { # define SQLITE_NOINLINE #endif +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1300 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_ReadWriteBarrier) +# else +# include +# endif +# endif +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -7980,10 +8990,9 @@ struct sqlite3_rtree_query_info { #endif /* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 @@ -8137,6 +9146,32 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # define NEVER(X) (X) #endif +/* +** Declarations used for tracing the operating system interfaces. +*/ +#if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) + extern int sqlite3OSTrace; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +# define SQLITE_HAVE_OS_TRACE +#else +# define OSTRACE(X) +# undef SQLITE_HAVE_OS_TRACE +#endif + +/* +** Is the sqlite3ErrName() function needed in the build? Currently, +** it is needed by "mutex_w32.c" (when debugging), "os_win.c" (when +** OSTRACE is enabled), and by several "test*.c" files (which are +** compiled using SQLITE_TEST). +*/ +#if defined(SQLITE_HAVE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) +# define SQLITE_NEED_ERR_NAME +#else +# undef SQLITE_NEED_ERR_NAME +#endif + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -8504,6 +9539,16 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS #endif +/* +** The default initial allocation for the pagecache when using separate +** pagecaches for each database connection. A positive number is the +** number of pages. A negative number N translations means that a buffer +** of -1024*N bytes is allocated and used for as many pages as it will hold. +*/ +#ifndef SQLITE_DEFAULT_PCACHE_INITSZ +# define SQLITE_DEFAULT_PCACHE_INITSZ 100 +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it @@ -8613,7 +9658,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantaties are suppose to be estimates, +** of 40. However, since LogEst quantities are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** ** "LogEst" is short for "Logarithmic Estimate". @@ -8632,6 +9677,20 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ */ typedef INT16_TYPE LogEst; +/* +** Set the SQLITE_PTRSIZE macro to the number of bytes in a pointer +*/ +#ifndef SQLITE_PTRSIZE +# if defined(__SIZEOF_POINTER__) +# define SQLITE_PTRSIZE __SIZEOF_POINTER__ +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(_M_ARM) || defined(__arm__) || defined(__x86) +# define SQLITE_PTRSIZE 4 +# else +# define SQLITE_PTRSIZE 8 +# endif +#endif + /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. @@ -8725,7 +9784,9 @@ SQLITE_PRIVATE const int sqlite3one; # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) + || defined(__sun) \ + || defined(__FreeBSD__) \ + || defined(__DragonFly__) # define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 @@ -8844,8 +9905,8 @@ struct BusyHandler { #define SQLITE_WSD const #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) -SQLITE_API int sqlite3_wsd_init(int N, int J); -SQLITE_API void *sqlite3_wsd_find(void *K, int L); +SQLITE_API int SQLITE_STDCALL sqlite3_wsd_init(int N, int J); +SQLITE_API void *SQLITE_STDCALL sqlite3_wsd_find(void *K, int L); #else #define SQLITE_WSD #define GLOBAL(t,v) v @@ -8949,7 +10010,7 @@ typedef struct With With; /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. */ -#define SQLITE_N_BTREE_META 10 +#define SQLITE_N_BTREE_META 16 /* ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise @@ -9003,10 +10064,8 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree*); -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); -#endif SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); @@ -9064,6 +10123,11 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); ** For example, the free-page-count field is located at byte offset 36 of ** the database file header. The incr-vacuum-flag field is located at ** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". */ #define BTREE_FREE_PAGE_COUNT 0 #define BTREE_SCHEMA_VERSION 1 @@ -9074,12 +10138,23 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 #define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* ** Values that may be OR'd together to form the second argument of an ** sqlite3BtreeCursorHints() call. +** +** The BTREE_BULKLOAD flag is set on index cursors when the index is going +** to be filled with content that is already in sorted order. +** +** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or +** OP_SeekLE opcodes for a range search, but where the range of entries +** selected will all have the same key. In other words, the cursor will +** be used only for equality key searches. +** */ -#define BTREE_BULKLOAD 0x00000001 +#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ +#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ SQLITE_PRIVATE int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ @@ -9101,7 +10176,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ); SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, int); SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, int nZero, int bias, int seekResult); @@ -9125,7 +10200,11 @@ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); +#endif SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); @@ -9237,13 +10316,14 @@ struct VdbeOp { int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ - union { /* fourth parameter */ + union p4union { /* fourth parameter */ int i; /* Integer value if p4type==P4_INT32 */ void *p; /* Generic pointer */ char *z; /* Pointer to data for string (char array) types */ i64 *pI64; /* Used when p4type is P4_INT64 */ double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ @@ -9310,6 +10390,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_FUNCCTX (-20) /* P4 is a pointer to an sqlite3_context object */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -9352,42 +10433,42 @@ typedef struct VdbeOpList VdbeOpList; /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ /* See the mkopcodeh.awk script for details */ -#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Savepoint 2 -#define OP_AutoCommit 3 -#define OP_Transaction 4 -#define OP_SorterNext 5 -#define OP_PrevIfOpen 6 -#define OP_NextIfOpen 7 -#define OP_Prev 8 -#define OP_Next 9 -#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_Checkpoint 11 -#define OP_JournalMode 12 -#define OP_Vacuum 13 -#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 16 -#define OP_Gosub 17 -#define OP_Return 18 +#define OP_Savepoint 1 +#define OP_AutoCommit 2 +#define OP_Transaction 3 +#define OP_SorterNext 4 +#define OP_PrevIfOpen 5 +#define OP_NextIfOpen 6 +#define OP_Prev 7 +#define OP_Next 8 +#define OP_Checkpoint 9 +#define OP_JournalMode 10 +#define OP_Vacuum 11 +#define OP_VFilter 12 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 13 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 14 +#define OP_Gosub 15 +#define OP_Return 16 +#define OP_InitCoroutine 17 +#define OP_EndCoroutine 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_InitCoroutine 20 -#define OP_EndCoroutine 21 -#define OP_Yield 22 -#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 24 -#define OP_Integer 25 /* synopsis: r[P2]=P1 */ -#define OP_Int64 26 /* synopsis: r[P2]=P4 */ -#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ -#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 36 +#define OP_Yield 20 +#define OP_HaltIfNull 21 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 22 +#define OP_Integer 23 /* synopsis: r[P2]=P1 */ +#define OP_Int64 24 /* synopsis: r[P2]=P4 */ +#define OP_String 25 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 26 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 27 /* synopsis: r[P1]=NULL */ +#define OP_Blob 28 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 29 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 30 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 31 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 32 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 33 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 34 +#define OP_Function0 35 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Function 36 /* synopsis: r[P3]=func(r[P2@P5]) */ #define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ #define OP_MustBeInt 38 #define OP_RealAffinity 39 @@ -9413,20 +10494,20 @@ typedef struct VdbeOpList VdbeOpList; #define OP_SequenceTest 59 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ #define OP_OpenPseudo 60 /* synopsis: P3 columns in r[P2] */ #define OP_Close 61 -#define OP_SeekLT 62 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLE 63 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGE 64 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGT 65 /* synopsis: key=r[P3@P4] */ -#define OP_Seek 66 /* synopsis: intkey=r[P2] */ -#define OP_NoConflict 67 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 68 /* synopsis: key=r[P3@P4] */ -#define OP_Found 69 /* synopsis: key=r[P3@P4] */ -#define OP_NotExists 70 /* synopsis: intkey=r[P3] */ +#define OP_ColumnsUsed 62 +#define OP_SeekLT 63 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 64 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 65 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 66 /* synopsis: key=r[P3@P4] */ +#define OP_Seek 67 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 68 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 69 /* synopsis: key=r[P3@P4] */ +#define OP_Found 70 /* synopsis: key=r[P3@P4] */ #define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_Sequence 73 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 74 /* synopsis: r[P2]=rowid */ -#define OP_Insert 75 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_NotExists 73 /* synopsis: intkey=r[P3] */ +#define OP_Sequence 74 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 75 /* synopsis: r[P2]=rowid */ #define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ @@ -9435,7 +10516,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ #define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ -#define OP_InsertInt 84 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Insert 84 /* synopsis: intkey=r[P3] data=r[P2] */ #define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 goto P2 */ -#define OP_IfNeg 138 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ -#define OP_IfZero 139 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ -#define OP_AggFinal 140 /* synopsis: accum=r[P1] N=P2 */ -#define OP_IncrVacuum 141 -#define OP_Expire 142 -#define OP_TableLock 143 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 144 -#define OP_VCreate 145 -#define OP_VDestroy 146 -#define OP_VOpen 147 -#define OP_VColumn 148 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VNext 149 -#define OP_VRename 150 -#define OP_Pagecount 151 -#define OP_MaxPgcnt 152 -#define OP_Init 153 /* synopsis: Start at P2 */ -#define OP_Noop 154 -#define OP_Explain 155 +#define OP_Param 134 +#define OP_FkCounter 135 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 136 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_MemMax 137 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 138 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_SetIfNotPos 139 /* synopsis: if r[P1]<=0 then r[P2]=P3 */ +#define OP_IfNotZero 140 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ +#define OP_DecrJumpZero 141 /* synopsis: if (--r[P1])==0 goto P2 */ +#define OP_JumpZeroIncr 142 /* synopsis: if (r[P1]++)==0 ) goto P2 */ +#define OP_AggStep0 143 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep 144 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggFinal 145 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 146 +#define OP_Expire 147 +#define OP_TableLock 148 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 149 +#define OP_VCreate 150 +#define OP_VDestroy 151 +#define OP_VOpen 152 +#define OP_VColumn 153 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 154 +#define OP_VRename 155 +#define OP_Pagecount 156 +#define OP_MaxPgcnt 157 +#define OP_Init 158 /* synopsis: Start at P2 */ +#define OP_Noop 159 +#define OP_Explain 160 /* Properties such as "out2" or "jump" that are specified in @@ -9514,33 +10600,33 @@ typedef struct VdbeOpList VdbeOpList; ** are encoded into bitvectors as follows: */ #define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ -#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */ -#define OPFLG_IN1 0x0004 /* in1: P1 is an input */ -#define OPFLG_IN2 0x0008 /* in2: P2 is an input */ -#define OPFLG_IN3 0x0010 /* in3: P3 is an input */ -#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */ -#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */ +#define OPFLG_IN1 0x0002 /* in1: P1 is an input */ +#define OPFLG_IN2 0x0004 /* in2: P2 is an input */ +#define OPFLG_IN3 0x0008 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x0010 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ -/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00,\ -/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x01, 0x04, 0x05, 0x10,\ -/* 24 */ 0x00, 0x02, 0x02, 0x02, 0x02, 0x00, 0x02, 0x02,\ -/* 32 */ 0x00, 0x00, 0x20, 0x00, 0x00, 0x04, 0x05, 0x04,\ -/* 40 */ 0x04, 0x00, 0x00, 0x01, 0x01, 0x05, 0x05, 0x00,\ -/* 48 */ 0x00, 0x00, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00,\ -/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x11,\ -/* 64 */ 0x11, 0x11, 0x08, 0x11, 0x11, 0x11, 0x11, 0x4c,\ -/* 72 */ 0x4c, 0x02, 0x02, 0x00, 0x05, 0x05, 0x15, 0x15,\ -/* 80 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x4c, 0x4c, 0x4c,\ -/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x00,\ -/* 96 */ 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,\ -/* 104 */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x08, 0x08, 0x00,\ -/* 112 */ 0x02, 0x01, 0x01, 0x01, 0x01, 0x02, 0x00, 0x00,\ -/* 120 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x02, 0x00, 0x01,\ -/* 136 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x01, 0x00, 0x00,\ -/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02,\ -/* 152 */ 0x02, 0x01, 0x00, 0x00,} +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,\ +/* 8 */ 0x01, 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01,\ +/* 16 */ 0x02, 0x01, 0x02, 0x12, 0x03, 0x08, 0x00, 0x10,\ +/* 24 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 32 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x02,\ +/* 40 */ 0x02, 0x00, 0x00, 0x01, 0x01, 0x03, 0x03, 0x00,\ +/* 48 */ 0x00, 0x00, 0x10, 0x10, 0x08, 0x00, 0x00, 0x00,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09,\ +/* 64 */ 0x09, 0x09, 0x09, 0x04, 0x09, 0x09, 0x09, 0x26,\ +/* 72 */ 0x26, 0x09, 0x10, 0x10, 0x03, 0x03, 0x0b, 0x0b,\ +/* 80 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x00, 0x26, 0x26, 0x26,\ +/* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ +/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 104 */ 0x10, 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04,\ +/* 112 */ 0x00, 0x10, 0x01, 0x01, 0x01, 0x01, 0x10, 0x00,\ +/* 120 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x00, 0x06, 0x23, 0x0b, 0x01, 0x10, 0x10, 0x00,\ +/* 136 */ 0x01, 0x04, 0x03, 0x06, 0x03, 0x03, 0x03, 0x00,\ +/* 144 */ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x01, 0x00, 0x10, 0x10, 0x01, 0x00,\ +/* 160 */ 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -9553,11 +10639,16 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); @@ -9599,6 +10690,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); @@ -9668,6 +10760,12 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); # define VDBE_OFFSET_LINENO(x) 0 #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +#else +# define sqlite3VdbeScanStatus(a,b,c,d,e) +#endif + #endif /************** End of vdbe.h ************************************************/ @@ -9794,6 +10892,9 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); @@ -9848,7 +10949,10 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); /* Functions used to query pager state and configuration. */ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); -SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); @@ -9864,6 +10968,8 @@ SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); #endif @@ -9937,14 +11043,14 @@ struct PgHdr { }; /* Bit values for PgHdr.flags */ -#define PGHDR_DIRTY 0x002 /* Page has changed */ -#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before - ** writing this page to the database */ -#define PGHDR_NEED_READ 0x008 /* Content is unread */ -#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ -#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ - -#define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ +#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ +#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ +#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_NEED_READ 0x010 /* Content is unread */ +#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); @@ -10051,6 +11157,10 @@ SQLITE_PRIVATE void sqlite3PcacheStats(int*,int*,int*,int*); SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); +/* Return the header size */ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void); +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); + #endif /* _PCACHE_H_ */ /************** End of pcache.h **********************************************/ @@ -10583,6 +11693,7 @@ struct sqlite3 { int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -10602,11 +11713,13 @@ struct sqlite3 { u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ + u8 imposterTable; /* Building an imposter table */ } init; int nVdbeActive; /* Number of VDBEs currently running */ int nVdbeRead; /* Number of active VDBEs that read or write */ int nVdbeWrite; /* Number of active VDBEs that read and write */ int nVdbeExec; /* Number of nested calls to VdbeExec() */ + int nVDestroy; /* Number of active OP_VDestroy operations */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ void (*xTrace)(void*,const char*); /* Trace function */ @@ -10684,7 +11797,8 @@ struct sqlite3 { /* ** A macro to discover the encoding of a database. */ -#define ENC(db) ((db)->aDb[0].pSchema->enc) +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) +#define ENC(db) ((db)->enc) /* ** Possible values for the sqlite3.flags. @@ -10719,6 +11833,8 @@ struct sqlite3 { #define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ +#define SQLITE_CellSizeCk 0x10000000 /* Check btree cell sizes on load */ /* @@ -10737,7 +11853,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -10813,18 +11929,20 @@ struct FuncDestructor { ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There ** are assert() statements in the code to verify this. */ -#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ -#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -10840,6 +11958,12 @@ struct FuncDestructor { ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -10860,11 +11984,14 @@ struct FuncDestructor { #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ @@ -10908,6 +12035,7 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -10953,6 +12081,7 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. @@ -10966,9 +12095,9 @@ struct CollSeq { ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. And the NONE type is first. +** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_NONE 'A' +#define SQLITE_AFF_BLOB 'A' #define SQLITE_AFF_TEXT 'B' #define SQLITE_AFF_NUMERIC 'C' #define SQLITE_AFF_INTEGER 'D' @@ -11049,34 +12178,8 @@ struct VTable { }; /* -** Each SQL table is represented in memory by an instance of the -** following structure. -** -** Table.zName is the name of the table. The case of the original -** CREATE TABLE statement is stored, but case is not significant for -** comparisons. -** -** Table.nCol is the number of columns in this table. Table.aCol is a -** pointer to an array of Column structures, one for each column. -** -** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of -** the column that is that key. Otherwise Table.iPKey is negative. Note -** that the datatype of the PRIMARY KEY must be INTEGER for this field to -** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of -** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid -** is generated for each row of the table. TF_HasPrimaryKey is set if -** the table has any PRIMARY KEY, INTEGER or otherwise. -** -** Table.tnum is the page number for the root BTree page of the table in the -** database file. If Table.iDb is the index of the database table backend -** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that -** holds temporary tables and indices. If TF_Ephemeral is set -** then the table is stored in a file that is automatically deleted -** when the VDBE cursor to the table is closed. In this case Table.tnum -** refers VDBE cursor number that holds the table open, not to the root -** page number. Transient tables are used to hold the results of a -** sub-query that appears instead of a real table name in the FROM clause -** of a SELECT statement. +** The schema for each SQL table and view is represented in memory +** by an instance of the following structure. */ struct Table { char *zName; /* Name of the table or view */ @@ -11085,14 +12188,13 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ -#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ -#endif - LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ - int tnum; /* Root BTree node for this table (see note above) */ - i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + /* ... also used as column name list in a VIEW */ + int tnum; /* Root BTree page for this table */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ @@ -11104,7 +12206,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* Text of all module args. [0] is module name */ + char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -11114,13 +12216,21 @@ struct Table { /* ** Allowed values for Table.tabFlags. +** +** TF_OOOHidden applies to virtual tables that have hidden columns that are +** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING +** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, +** the TF_OOOHidden attribute would apply in this case. Such tables require +** special handling during INSERT processing. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ +#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ /* @@ -11138,6 +12248,7 @@ struct Table { /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) +#define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) /* ** Each foreign key constraint is an instance of the following structure. @@ -11296,6 +12407,14 @@ struct UnpackedRecord { ** and the value of Index.onError indicate the which conflict resolution ** algorithm to employ whenever an attempt is made to insert a non-unique ** element. +** +** While parsing a CREATE TABLE or CREATE INDEX statement in order to +** generate VDBE code (as opposed to parsing one read from an sqlite_master +** table as part of parsing an existing database schema), transient instances +** of this structure may be created. In this case the Index.tnum variable is +** used to store the address of a VDBE instruction, not a database page +** number (it cannot - the database page is not allocated until the VDBE +** program is executed). See convertToWithoutRowidTable() for details. */ struct Index { char *zName; /* Name of this index */ @@ -11308,7 +12427,7 @@ struct Index { u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ - KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ + ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -11319,12 +12438,14 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ - tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; @@ -11341,6 +12462,12 @@ struct Index { /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the @@ -11522,7 +12649,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ @@ -11537,7 +12664,7 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ @@ -11556,7 +12683,15 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_Constant 0x080000 /* Node is a constant */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ + +/* +** Combinations of two or more EP_* flags +*/ +#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ /* ** These macros can be used to test, set, or clear bits in the @@ -11714,11 +12849,15 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ + struct { + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + } fg; #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -11726,8 +12865,11 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ - Index *pIndex; /* Index structure corresponding to zIndex, if any */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ } a[1]; /* One entry for each identifier on the list */ }; @@ -11755,12 +12897,13 @@ struct SrcList { #define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ - /* 0x0080 // not currently used */ +#define WHERE_NO_AUTOINDEX 0x0080 /* Disallow automatic indexes */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -11813,6 +12956,7 @@ struct NameContext { #define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ #define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* @@ -11862,18 +13006,20 @@ struct Select { ** "Select Flag". */ #define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_Resolved 0x0002 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0004 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x0040 /* Part of a compound query */ -#define SF_Values 0x0080 /* Synthesized from VALUES clause */ - /* 0x0100 NOT USED */ -#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ -#define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ -#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ +#define SF_All 0x0002 /* Includes the ALL keyword */ +#define SF_Resolved 0x0004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0008 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0010 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0020 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0040 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0080 /* Part of a compound query */ +#define SF_Values 0x0100 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0200 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0400 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x0800 /* Need convertCompoundSelectToSubquery() */ #define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x2000 /* The recursive part of a recursive CTE */ +#define SF_Converted 0x4000 /* By convertCompoundSelectToSubquery() */ /* @@ -12080,7 +13226,7 @@ struct Parse { int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ - int iPartIdxTab; /* Table corresponding to a partial index */ + int iSelfTab; /* Table of an index whose exprs are being coded */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ @@ -12115,7 +13261,6 @@ struct Parse { Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ - int addrSkipPK; /* Address of instruction to skip PRIMARY KEY index */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ @@ -12192,7 +13337,8 @@ struct AuthContext { #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ -#define OPFLAG_P2ISREG 0x02 /* P2 to OP_Open** is a register number */ +#define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ +#define OPFLAG_P2ISREG 0x04 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ /* @@ -12251,7 +13397,7 @@ struct Trigger { * orconf -> stores the ON CONFLICT algorithm * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then * this stores a pointer to the SELECT statement. Otherwise NULL. - * target -> A token holding the quoted name of the table to insert into. + * zTarget -> Dequoted name of the table to insert into. * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then * this stores values to be inserted. Otherwise NULL. * pIdList -> If this is an INSERT INTO ... () VALUES ... @@ -12259,12 +13405,12 @@ struct Trigger { * inserted into. * * (op == TK_DELETE) - * target -> A token holding the quoted name of the table to delete from. + * zTarget -> Dequoted name of the table to delete from. * pWhere -> The WHERE clause of the DELETE statement if one is specified. * Otherwise NULL. * * (op == TK_UPDATE) - * target -> A token holding the quoted name of the table to update rows of. + * zTarget -> Dequoted name of the table to update. * pWhere -> The WHERE clause of the UPDATE statement if one is specified. * Otherwise NULL. * pExprList -> A list of the columns to update and the expressions to update @@ -12276,8 +13422,8 @@ struct TriggerStep { u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ - Select *pSelect; /* SELECT statment or RHS of INSERT INTO .. SELECT ... */ - Token target; /* Target table for DELETE, UPDATE, INSERT */ + Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ + char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE. */ IdList *pIdList; /* Column names for INSERT */ @@ -12310,8 +13456,7 @@ struct StrAccum { char *zText; /* The string collected so far */ int nChar; /* Length of the string so far */ int nAlloc; /* Amount of space allocated in zText */ - int mxAlloc; /* Maximum allowed string length */ - u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ + int mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ }; #define STRACCUM_NOMEM 1 @@ -12359,6 +13504,7 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ + u32 szPma; /* Maximum Sorter PMA size */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ @@ -12414,9 +13560,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -12448,7 +13596,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zErr; /* Error message for circular references */ + const char *zCteErr; /* Error message for circular references */ } a[1]; }; @@ -12494,7 +13642,7 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); ** the SQLITE_ENABLE_FTS4 macro to serve as an alias for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 +# define SQLITE_ENABLE_FTS3 1 #endif /* @@ -12528,7 +13676,9 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) #endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); +#endif /* ** Internal function prototypes @@ -12556,7 +13706,9 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); +#endif SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* @@ -12592,11 +13744,21 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); SQLITE_PRIVATE int sqlite3MutexInit(void); SQLITE_PRIVATE int sqlite3MutexEnd(void); #endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) +SQLITE_PRIVATE void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier() +#endif -SQLITE_PRIVATE int sqlite3StatusValue(int); -SQLITE_PRIVATE void sqlite3StatusAdd(int, int); +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); +SQLITE_PRIVATE void sqlite3StatusUp(int, int); +SQLITE_PRIVATE void sqlite3StatusDown(int, int); SQLITE_PRIVATE void sqlite3StatusSet(int, int); +/* Access to mutexes used by sqlite3_status() */ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void); + #ifndef SQLITE_OMIT_FLOATING_POINT SQLITE_PRIVATE int sqlite3IsNaN(double); #else @@ -12619,8 +13781,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...); -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); #endif #if defined(SQLITE_TEST) @@ -12628,17 +13789,13 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif #if defined(SQLITE_DEBUG) -SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView*,u8); -SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView*); -SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView*, const char*, ...); -SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView*, const char*, u8); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); #endif -SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*, ...); +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); SQLITE_PRIVATE int sqlite3Dequote(char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); @@ -12658,9 +13815,11 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); @@ -12669,6 +13828,8 @@ SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); @@ -12695,11 +13856,14 @@ SQLITE_PRIVATE int sqlite3FaultSim(int); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec*, u32); SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); +#endif SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); @@ -12707,7 +13871,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); -SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); @@ -12737,6 +13901,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*) SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); @@ -12767,6 +13932,10 @@ SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ +#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ +#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); @@ -12782,11 +13951,13 @@ SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); @@ -12803,8 +13974,10 @@ SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3PrngSaveState(void); SQLITE_PRIVATE void sqlite3PrngRestoreState(void); +#endif SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); @@ -12817,12 +13990,14 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); -SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, @@ -12880,6 +14055,7 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) @@ -12889,6 +14065,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif @@ -12952,7 +14129,7 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); #define putVarint sqlite3PutVarint -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); @@ -12965,7 +14142,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); -#if defined(SQLITE_TEST) +#if defined(SQLITE_NEED_ERR_NAME) SQLITE_PRIVATE const char *sqlite3ErrName(int); #endif @@ -12974,7 +14151,7 @@ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); @@ -13021,8 +14198,10 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -13059,7 +14238,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); -SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, char*, int, int); +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum*,const char*,int); SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum*,const char*); SQLITE_PRIVATE void sqlite3AppendChar(StrAccum*,int,char); @@ -13131,6 +14310,8 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); @@ -13243,12 +14424,11 @@ SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); SQLITE_PRIVATE int sqlite3MemJournalSize(void); SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *); +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 -SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p); SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *); SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); #else - #define sqlite3ExprSetHeight(x,y) #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif @@ -13278,7 +14458,7 @@ SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); #ifdef SQLITE_ENABLE_IOTRACE # define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe*); -SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); +SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); #else # define IOTRACE(A) # define sqlite3VdbeIOTraceSql(X) @@ -13334,6 +14514,10 @@ SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); #endif +#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); +#endif + #endif /* _SQLITEINT_H_ */ /************** End of sqliteInt.h *******************************************/ @@ -13352,6 +14536,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); ** ** This file contains definitions of global variables and constants. */ +/* #include "sqliteInt.h" */ /* An array to map all upper-case characters into their corresponding ** lower-case character. @@ -13385,16 +14570,16 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, /* 3x */ 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, /* 4x */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, /* 5x */ - 96, 97, 66, 67, 68, 69, 70, 71, 72, 73,106,107,108,109,110,111, /* 6x */ - 112, 81, 82, 83, 84, 85, 86, 87, 88, 89,122,123,124,125,126,127, /* 7x */ + 96, 97, 98, 99,100,101,102,103,104,105,106,107,108,109,110,111, /* 6x */ + 112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127, /* 7x */ 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, /* 8x */ - 144,145,146,147,148,149,150,151,152,153,154,155,156,157,156,159, /* 9x */ + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, /* 9x */ 160,161,162,163,164,165,166,167,168,169,170,171,140,141,142,175, /* Ax */ 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, /* Bx */ 192,129,130,131,132,133,134,135,136,137,202,203,204,205,206,207, /* Cx */ 208,145,146,147,148,149,150,151,152,153,218,219,220,221,222,223, /* Dx */ - 224,225,162,163,164,165,166,167,168,169,232,203,204,205,206,207, /* Ex */ - 239,240,241,242,243,244,245,246,247,248,249,219,220,221,222,255, /* Fx */ + 224,225,162,163,164,165,166,167,168,169,234,235,236,237,238,239, /* Ex */ + 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, /* Fx */ #endif }; @@ -13474,15 +14659,30 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** ** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled ** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. */ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ #ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 #endif +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. +*/ +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 250 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -13510,9 +14710,10 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ - 0, /* nPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ + SQLITE_SORTER_PMASZ, /* szPma */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ @@ -13568,13 +14769,14 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { ** ** IMPORTANT: Changing the pending byte to any value other than ** 0x40000000 results in an incompatible database file format! -** Changing the pending byte during operating results in undefined -** and dileterious behavior. +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. */ #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; #endif +/* #include "opcodes.h" */ /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is ** created by mkopcodeh.awk during compilation. Data is obtained @@ -13603,6 +14805,7 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* #include "sqliteInt.h" */ /* ** An array of names of all compile-time options. This array should @@ -13619,88 +14822,100 @@ static const char * const azCompileOpt[] = { #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) -#ifdef SQLITE_32BIT_ROWID +#if SQLITE_32BIT_ROWID "32BIT_ROWID", #endif -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +#if SQLITE_4_BYTE_ALIGNED_MALLOC "4_BYTE_ALIGNED_MALLOC", #endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE +#if SQLITE_CASE_SENSITIVE_LIKE "CASE_SENSITIVE_LIKE", #endif -#ifdef SQLITE_CHECK_PAGES +#if SQLITE_CHECK_PAGES "CHECK_PAGES", #endif -#ifdef SQLITE_COVERAGE_TEST +#if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_DEBUG +#if SQLITE_DEBUG "DEBUG", #endif -#ifdef SQLITE_DEFAULT_LOCKING_MODE +#if SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif #if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif -#ifdef SQLITE_DISABLE_DIRSYNC +#if SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif -#ifdef SQLITE_DISABLE_LFS +#if SQLITE_DISABLE_LFS "DISABLE_LFS", #endif -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif +#if SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif -#ifdef SQLITE_ENABLE_CEROD +#if SQLITE_ENABLE_CEROD "ENABLE_CEROD", #endif -#ifdef SQLITE_ENABLE_COLUMN_METADATA +#if SQLITE_ENABLE_COLUMN_METADATA "ENABLE_COLUMN_METADATA", #endif -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT +#if SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", +#endif +#if SQLITE_ENABLE_EXPENSIVE_ASSERT "ENABLE_EXPENSIVE_ASSERT", #endif -#ifdef SQLITE_ENABLE_FTS1 +#if SQLITE_ENABLE_FTS1 "ENABLE_FTS1", #endif -#ifdef SQLITE_ENABLE_FTS2 +#if SQLITE_ENABLE_FTS2 "ENABLE_FTS2", #endif -#ifdef SQLITE_ENABLE_FTS3 +#if SQLITE_ENABLE_FTS3 "ENABLE_FTS3", #endif -#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +#if SQLITE_ENABLE_FTS3_PARENTHESIS "ENABLE_FTS3_PARENTHESIS", #endif -#ifdef SQLITE_ENABLE_FTS4 +#if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", #endif -#ifdef SQLITE_ENABLE_ICU +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif +#if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif -#ifdef SQLITE_ENABLE_IOTRACE +#if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif +#if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif -#ifdef SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), #endif -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +#if SQLITE_ENABLE_MEMORY_MANAGEMENT "ENABLE_MEMORY_MANAGEMENT", #endif -#ifdef SQLITE_ENABLE_MEMSYS3 +#if SQLITE_ENABLE_MEMSYS3 "ENABLE_MEMSYS3", #endif -#ifdef SQLITE_ENABLE_MEMSYS5 +#if SQLITE_ENABLE_MEMSYS5 "ENABLE_MEMSYS5", #endif -#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK +#if SQLITE_ENABLE_OVERSIZE_CELL_CHECK "ENABLE_OVERSIZE_CELL_CHECK", #endif -#ifdef SQLITE_ENABLE_RTREE +#if SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif #if defined(SQLITE_ENABLE_STAT4) @@ -13708,31 +14923,31 @@ static const char * const azCompileOpt[] = { #elif defined(SQLITE_ENABLE_STAT3) "ENABLE_STAT3", #endif -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +#if SQLITE_ENABLE_UNLOCK_NOTIFY "ENABLE_UNLOCK_NOTIFY", #endif -#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT +#if SQLITE_ENABLE_UPDATE_DELETE_LIMIT "ENABLE_UPDATE_DELETE_LIMIT", #endif -#ifdef SQLITE_HAS_CODEC +#if SQLITE_HAS_CODEC "HAS_CODEC", #endif -#ifdef SQLITE_HAVE_ISNAN +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN "HAVE_ISNAN", #endif -#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX +#if SQLITE_HOMEGROWN_RECURSIVE_MUTEX "HOMEGROWN_RECURSIVE_MUTEX", #endif -#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS +#if SQLITE_IGNORE_AFP_LOCK_ERRORS "IGNORE_AFP_LOCK_ERRORS", #endif -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS +#if SQLITE_IGNORE_FLOCK_LOCK_ERRORS "IGNORE_FLOCK_LOCK_ERRORS", #endif #ifdef SQLITE_INT64_TYPE "INT64_TYPE", #endif -#ifdef SQLITE_LOCK_TRACE +#if SQLITE_LOCK_TRACE "LOCK_TRACE", #endif #if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) @@ -13741,226 +14956,226 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif -#ifdef SQLITE_MEMDEBUG +#if SQLITE_MEMDEBUG "MEMDEBUG", #endif -#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +#if SQLITE_MIXED_ENDIAN_64BIT_FLOAT "MIXED_ENDIAN_64BIT_FLOAT", #endif -#ifdef SQLITE_NO_SYNC +#if SQLITE_NO_SYNC "NO_SYNC", #endif -#ifdef SQLITE_OMIT_ALTERTABLE +#if SQLITE_OMIT_ALTERTABLE "OMIT_ALTERTABLE", #endif -#ifdef SQLITE_OMIT_ANALYZE +#if SQLITE_OMIT_ANALYZE "OMIT_ANALYZE", #endif -#ifdef SQLITE_OMIT_ATTACH +#if SQLITE_OMIT_ATTACH "OMIT_ATTACH", #endif -#ifdef SQLITE_OMIT_AUTHORIZATION +#if SQLITE_OMIT_AUTHORIZATION "OMIT_AUTHORIZATION", #endif -#ifdef SQLITE_OMIT_AUTOINCREMENT +#if SQLITE_OMIT_AUTOINCREMENT "OMIT_AUTOINCREMENT", #endif -#ifdef SQLITE_OMIT_AUTOINIT +#if SQLITE_OMIT_AUTOINIT "OMIT_AUTOINIT", #endif -#ifdef SQLITE_OMIT_AUTOMATIC_INDEX +#if SQLITE_OMIT_AUTOMATIC_INDEX "OMIT_AUTOMATIC_INDEX", #endif -#ifdef SQLITE_OMIT_AUTORESET +#if SQLITE_OMIT_AUTORESET "OMIT_AUTORESET", #endif -#ifdef SQLITE_OMIT_AUTOVACUUM +#if SQLITE_OMIT_AUTOVACUUM "OMIT_AUTOVACUUM", #endif -#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION +#if SQLITE_OMIT_BETWEEN_OPTIMIZATION "OMIT_BETWEEN_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_BLOB_LITERAL +#if SQLITE_OMIT_BLOB_LITERAL "OMIT_BLOB_LITERAL", #endif -#ifdef SQLITE_OMIT_BTREECOUNT +#if SQLITE_OMIT_BTREECOUNT "OMIT_BTREECOUNT", #endif -#ifdef SQLITE_OMIT_BUILTIN_TEST +#if SQLITE_OMIT_BUILTIN_TEST "OMIT_BUILTIN_TEST", #endif -#ifdef SQLITE_OMIT_CAST +#if SQLITE_OMIT_CAST "OMIT_CAST", #endif -#ifdef SQLITE_OMIT_CHECK +#if SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -#ifdef SQLITE_OMIT_COMPLETE +#if SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif -#ifdef SQLITE_OMIT_COMPOUND_SELECT +#if SQLITE_OMIT_COMPOUND_SELECT "OMIT_COMPOUND_SELECT", #endif -#ifdef SQLITE_OMIT_CTE +#if SQLITE_OMIT_CTE "OMIT_CTE", #endif -#ifdef SQLITE_OMIT_DATETIME_FUNCS +#if SQLITE_OMIT_DATETIME_FUNCS "OMIT_DATETIME_FUNCS", #endif -#ifdef SQLITE_OMIT_DECLTYPE +#if SQLITE_OMIT_DECLTYPE "OMIT_DECLTYPE", #endif -#ifdef SQLITE_OMIT_DEPRECATED +#if SQLITE_OMIT_DEPRECATED "OMIT_DEPRECATED", #endif -#ifdef SQLITE_OMIT_DISKIO +#if SQLITE_OMIT_DISKIO "OMIT_DISKIO", #endif -#ifdef SQLITE_OMIT_EXPLAIN +#if SQLITE_OMIT_EXPLAIN "OMIT_EXPLAIN", #endif -#ifdef SQLITE_OMIT_FLAG_PRAGMAS +#if SQLITE_OMIT_FLAG_PRAGMAS "OMIT_FLAG_PRAGMAS", #endif -#ifdef SQLITE_OMIT_FLOATING_POINT +#if SQLITE_OMIT_FLOATING_POINT "OMIT_FLOATING_POINT", #endif -#ifdef SQLITE_OMIT_FOREIGN_KEY +#if SQLITE_OMIT_FOREIGN_KEY "OMIT_FOREIGN_KEY", #endif -#ifdef SQLITE_OMIT_GET_TABLE +#if SQLITE_OMIT_GET_TABLE "OMIT_GET_TABLE", #endif -#ifdef SQLITE_OMIT_INCRBLOB +#if SQLITE_OMIT_INCRBLOB "OMIT_INCRBLOB", #endif -#ifdef SQLITE_OMIT_INTEGRITY_CHECK +#if SQLITE_OMIT_INTEGRITY_CHECK "OMIT_INTEGRITY_CHECK", #endif -#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION +#if SQLITE_OMIT_LIKE_OPTIMIZATION "OMIT_LIKE_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION +#if SQLITE_OMIT_LOAD_EXTENSION "OMIT_LOAD_EXTENSION", #endif -#ifdef SQLITE_OMIT_LOCALTIME +#if SQLITE_OMIT_LOCALTIME "OMIT_LOCALTIME", #endif -#ifdef SQLITE_OMIT_LOOKASIDE +#if SQLITE_OMIT_LOOKASIDE "OMIT_LOOKASIDE", #endif -#ifdef SQLITE_OMIT_MEMORYDB +#if SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_OR_OPTIMIZATION +#if SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_PAGER_PRAGMAS +#if SQLITE_OMIT_PAGER_PRAGMAS "OMIT_PAGER_PRAGMAS", #endif -#ifdef SQLITE_OMIT_PRAGMA +#if SQLITE_OMIT_PRAGMA "OMIT_PRAGMA", #endif -#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +#if SQLITE_OMIT_PROGRESS_CALLBACK "OMIT_PROGRESS_CALLBACK", #endif -#ifdef SQLITE_OMIT_QUICKBALANCE +#if SQLITE_OMIT_QUICKBALANCE "OMIT_QUICKBALANCE", #endif -#ifdef SQLITE_OMIT_REINDEX +#if SQLITE_OMIT_REINDEX "OMIT_REINDEX", #endif -#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS +#if SQLITE_OMIT_SCHEMA_PRAGMAS "OMIT_SCHEMA_PRAGMAS", #endif -#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS +#if SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS "OMIT_SCHEMA_VERSION_PRAGMAS", #endif -#ifdef SQLITE_OMIT_SHARED_CACHE +#if SQLITE_OMIT_SHARED_CACHE "OMIT_SHARED_CACHE", #endif -#ifdef SQLITE_OMIT_SUBQUERY +#if SQLITE_OMIT_SUBQUERY "OMIT_SUBQUERY", #endif -#ifdef SQLITE_OMIT_TCL_VARIABLE +#if SQLITE_OMIT_TCL_VARIABLE "OMIT_TCL_VARIABLE", #endif -#ifdef SQLITE_OMIT_TEMPDB +#if SQLITE_OMIT_TEMPDB "OMIT_TEMPDB", #endif -#ifdef SQLITE_OMIT_TRACE +#if SQLITE_OMIT_TRACE "OMIT_TRACE", #endif -#ifdef SQLITE_OMIT_TRIGGER +#if SQLITE_OMIT_TRIGGER "OMIT_TRIGGER", #endif -#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION +#if SQLITE_OMIT_TRUNCATE_OPTIMIZATION "OMIT_TRUNCATE_OPTIMIZATION", #endif -#ifdef SQLITE_OMIT_UTF16 +#if SQLITE_OMIT_UTF16 "OMIT_UTF16", #endif -#ifdef SQLITE_OMIT_VACUUM +#if SQLITE_OMIT_VACUUM "OMIT_VACUUM", #endif -#ifdef SQLITE_OMIT_VIEW +#if SQLITE_OMIT_VIEW "OMIT_VIEW", #endif -#ifdef SQLITE_OMIT_VIRTUALTABLE +#if SQLITE_OMIT_VIRTUALTABLE "OMIT_VIRTUALTABLE", #endif -#ifdef SQLITE_OMIT_WAL +#if SQLITE_OMIT_WAL "OMIT_WAL", #endif -#ifdef SQLITE_OMIT_WSD +#if SQLITE_OMIT_WSD "OMIT_WSD", #endif -#ifdef SQLITE_OMIT_XFER_OPT +#if SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PERFORMANCE_TRACE +#if SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif -#ifdef SQLITE_PROXY_DEBUG +#if SQLITE_PROXY_DEBUG "PROXY_DEBUG", #endif -#ifdef SQLITE_RTREE_INT_ONLY +#if SQLITE_RTREE_INT_ONLY "RTREE_INT_ONLY", #endif -#ifdef SQLITE_SECURE_DELETE +#if SQLITE_SECURE_DELETE "SECURE_DELETE", #endif -#ifdef SQLITE_SMALL_STACK +#if SQLITE_SMALL_STACK "SMALL_STACK", #endif -#ifdef SQLITE_SOUNDEX +#if SQLITE_SOUNDEX "SOUNDEX", #endif -#ifdef SQLITE_SYSTEM_MALLOC +#if SQLITE_SYSTEM_MALLOC "SYSTEM_MALLOC", #endif -#ifdef SQLITE_TCL +#if SQLITE_TCL "TCL", #endif #if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif -#ifdef SQLITE_TEST +#if SQLITE_TEST "TEST", #endif #if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif -#ifdef SQLITE_USE_ALLOCA +#if SQLITE_USE_ALLOCA "USE_ALLOCA", #endif -#ifdef SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION "USER_AUTHENTICATION", #endif -#ifdef SQLITE_WIN32_MALLOC +#if SQLITE_WIN32_MALLOC "WIN32_MALLOC", #endif -#ifdef SQLITE_ZERO_MALLOC +#if SQLITE_ZERO_MALLOC "ZERO_MALLOC" #endif }; @@ -13972,8 +15187,15 @@ static const char * const azCompileOpt[] = { ** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix ** is not required for a match. */ -SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ +SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#if SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); @@ -13993,7 +15215,7 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ ** Return the N-th compile-time option string. If N is out of range, ** return a NULL pointer. */ -SQLITE_API const char *sqlite3_compileoption_get(int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N){ if( N>=0 && NnChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) @@ -14193,6 +15421,7 @@ struct Mem { } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ @@ -14207,6 +15436,12 @@ struct Mem { #endif }; +/* +** Size of struct Mem not including the Mem.zMalloc member or anything that +** follows. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) + /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** @@ -14291,14 +15526,16 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { - Mem *pOut; /* The return value is stored here */ - FuncDef *pFunc; /* Pointer to function information */ - Mem *pMem; /* Memory cell used to store aggregate context */ - Vdbe *pVdbe; /* The VM that owns this context */ - int iOp; /* Instruction number of OP_Function */ - int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip accumulator loading if true */ - u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ + Mem *pMem; /* Memory cell used to store aggregate context */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip accumulator loading if true */ + u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ + u8 argc; /* Number of arguments */ + sqlite3_value *argv[1]; /* Argument set */ }; /* @@ -14318,20 +15555,22 @@ struct Explain { */ typedef unsigned bft; /* Bit Field Type */ +typedef struct ScanStatus ScanStatus; +struct ScanStatus { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ + LogEst nEst; /* Estimated output rows per loop */ + char *zName; /* Name of table or index */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. ** ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare() ** is really a pointer to an instance of this structure. -** -** The Vdbe.inVtabMethod variable is set to non-zero for the duration of -** any virtual table method invocations made by the vdbe program. It is -** set to 2 for xDestroy method calls and 1 for all other methods. This -** variable is used for two purposes: to allow xDestroy methods to execute -** "DROP TABLE" statements and to prevent some nasty side effects of -** malloc failure when SQLite is invoked recursively by a virtual table -** method function. */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ @@ -14355,11 +15594,13 @@ struct Vdbe { u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ +#ifdef SQLITE_DEBUG + int rcApp; /* errcode set by sqlite3_result_error_code() */ +#endif u16 nResColumn; /* Number of columns in one row of the result set */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ bft explain:2; /* True if EXPLAIN present on SQL command */ - bft inVtabMethod:2; /* See comments above */ bft changeCntOn:1; /* True to update the change-counter */ bft expired:1; /* True if the VM needs to be recompiled */ bft runOnlyOnce:1; /* Automatically expire on reset */ @@ -14390,6 +15631,11 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif }; /* @@ -14403,6 +15649,7 @@ struct Vdbe { /* ** Function prototypes */ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); @@ -14514,10 +15761,32 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); */ typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { - int nowValue[10]; /* Current value */ - int mxValue[10]; /* Maximum value */ +#if SQLITE_PTRSIZE>4 + sqlite3_int64 nowValue[10]; /* Current value */ + sqlite3_int64 mxValue[10]; /* Maximum value */ +#else + u32 nowValue[10]; /* Current value */ + u32 mxValue[10]; /* Maximum value */ +#endif } sqlite3Stat = { {0,}, {0,} }; +/* +** Elements of sqlite3Stat[] are protected by either the memory allocator +** mutex, or by the pcache1 mutex. The following array determines which. +*/ +static const char statMutex[] = { + 0, /* SQLITE_STATUS_MEMORY_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_OVERFLOW */ + 0, /* SQLITE_STATUS_SCRATCH_USED */ + 0, /* SQLITE_STATUS_SCRATCH_OVERFLOW */ + 0, /* SQLITE_STATUS_MALLOC_SIZE */ + 0, /* SQLITE_STATUS_PARSER_STACK */ + 1, /* SQLITE_STATUS_PAGECACHE_SIZE */ + 0, /* SQLITE_STATUS_SCRATCH_SIZE */ + 0, /* SQLITE_STATUS_MALLOC_COUNT */ +}; + /* The "wsdStat" macro will resolve to the status information ** state vector. If writable static data is unsupported on the target, @@ -14534,33 +15803,60 @@ static SQLITE_WSD struct sqlite3StatType { #endif /* -** Return the current value of a status parameter. +** Return the current value of a status parameter. The caller must +** be holding the appropriate mutex. */ -SQLITE_PRIVATE int sqlite3StatusValue(int op){ +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int op){ wsdStatInit; assert( op>=0 && op=0 && op=0 && op=0 && opwsdStat.mxValue[op] ){ wsdStat.mxValue[op] = wsdStat.nowValue[op]; } } +SQLITE_PRIVATE void sqlite3StatusDown(int op, int N){ + wsdStatInit; + assert( N>=0 ); + assert( op>=0 && op=0 && op=0 && op=0 && opwsdStat.mxValue[op] ){ wsdStat.mxValue[op] = wsdStat.nowValue[op]; @@ -14569,28 +15865,50 @@ SQLITE_PRIVATE void sqlite3StatusSet(int op, int X){ /* ** Query status information. -** -** This implementation assumes that reading or writing an aligned -** 32-bit integer is an atomic operation. If that assumption is not true, -** then this routine is not threadsafe. */ -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ +SQLITE_API int SQLITE_STDCALL sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +){ + sqlite3_mutex *pMutex; wsdStatInit; if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + pMutex = statMutex[op] ? sqlite3Pcache1Mutex() : sqlite3MallocMutex(); + sqlite3_mutex_enter(pMutex); *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ wsdStat.mxValue[op] = wsdStat.nowValue[op]; } + sqlite3_mutex_leave(pMutex); + (void)pMutex; /* Prevent warning when SQLITE_THREADSAFE=0 */ return SQLITE_OK; } +SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + sqlite3_int64 iCur, iHwtr; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + rc = sqlite3_status64(op, &iCur, &iHwtr, resetFlag); + if( rc==0 ){ + *pCurrent = (int)iCur; + *pHighwater = (int)iHwtr; + } + return rc; +} /* ** Query status information for a single database connection */ -SQLITE_API int sqlite3_db_status( +SQLITE_API int SQLITE_STDCALL sqlite3_db_status( sqlite3 *db, /* The database connection whose status is desired */ int op, /* Status verb */ int *pCurrent, /* Write current value here */ @@ -14598,6 +15916,11 @@ SQLITE_API int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { @@ -14776,7 +16099,7 @@ SQLITE_API int sqlite3_db_status( ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** SQLite processes all times and dates as Julian Day numbers. The +** SQLite processes all times and dates as julian day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian ** calendar system. @@ -14791,7 +16114,7 @@ SQLITE_API int sqlite3_db_status( ** ** The Gregorian calendar system is used for all dates and times, ** even those that predate the Gregorian calendar. Historians usually -** use the Julian calendar for dates prior to 1582-10-15 and for some +** use the julian calendar for dates prior to 1582-10-15 and for some ** dates afterwards, depending on locale. Beware of this difference. ** ** The conversion algorithms are implemented based on descriptions @@ -14803,6 +16126,7 @@ SQLITE_API int sqlite3_db_status( ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ #include @@ -15063,7 +16387,7 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ } /* -** Attempt to parse the given string into a Julian Day Number. Return +** Attempt to parse the given string into a julian day number. Return ** the number of errors. ** ** The following are acceptable forms for the input string: @@ -15114,7 +16438,7 @@ static void computeYMD(DateTime *p){ A = Z + 1 + A - (A/4); B = A + 1524; C = (int)((B - 122.1)/365.25); - D = (36525*C)/100; + D = (36525*(C&32767))/100; E = (int)((B-D)/30.6001); X1 = (int)(30.6001*E); p->D = B - D - X1; @@ -15171,8 +16495,9 @@ static void clearYMD_HMS_TZ(DateTime *p){ ** already, check for an MSVC build environment that provides ** localtime_s(). */ -#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ - defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S \ + && defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#undef HAVE_LOCALTIME_S #define HAVE_LOCALTIME_S 1 #endif @@ -15192,8 +16517,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ */ static int osLocaltime(time_t *t, struct tm *pTm){ int rc; -#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ - && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S struct tm *pX; #if SQLITE_THREADSAFE>0 sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); @@ -15210,7 +16534,7 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #ifndef SQLITE_OMIT_BUILTIN_TEST if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; #endif -#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R +#if HAVE_LOCALTIME_R rc = localtime_r(t, pTm)==0; #else rc = localtime_s(pTm, t); @@ -15634,7 +16958,7 @@ static void dateFunc( ** %f ** fractional seconds SS.SSS ** %H hour 00-24 ** %j day of year 000-366 -** %J ** Julian day number +** %J ** julian day number ** %m month 01-12 ** %M minute 00-59 ** %s seconds since 1970-01-01 @@ -15654,8 +16978,10 @@ static void strftimeFunc( size_t i,j; char *z; sqlite3 *db; - const char *zFmt = (const char*)sqlite3_value_text(argv[0]); + const char *zFmt; char zBuf[100]; + if( argc==0 ) return; + zFmt = (const char*)sqlite3_value_text(argv[0]); if( zFmt==0 || isDate(context, argc-1, argv+1, &x) ) return; db = sqlite3_context_db_handle(context); for(i=0, n=1; zFmt[i]; i++, n++){ @@ -15849,7 +17175,7 @@ static void currentTimeFunc( iT = sqlite3StmtCurrentTime(context); if( iT<=0 ) return; t = iT/1000 - 10000*(sqlite3_int64)21086676; -#ifdef HAVE_GMTIME_R +#if HAVE_GMTIME_R pTm = gmtime_r(&t, &sNow); #else sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); @@ -15872,14 +17198,14 @@ static void currentTimeFunc( SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), @@ -15913,6 +17239,7 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ ** architectures. */ #define _SQLITE_OS_C_ 1 +/* #include "sqliteInt.h" */ #undef _SQLITE_OS_C_ /* @@ -16207,7 +17534,7 @@ static sqlite3_vfs * SQLITE_WSD vfsList = 0; ** Locate a VFS by name. If no name is given, simply return the ** first VFS on the list. */ -SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ +SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfs){ sqlite3_vfs *pVfs = 0; #if SQLITE_THREADSAFE sqlite3_mutex *mutex; @@ -16253,12 +17580,16 @@ static void vfsUnlink(sqlite3_vfs *pVfs){ ** VFS multiple times. The new VFS becomes the default if makeDflt is ** true. */ -SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ MUTEX_LOGIC(sqlite3_mutex *mutex;) #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); @@ -16277,7 +17608,7 @@ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ /* ** Unregister a VFS so that it is no longer accessible. */ -SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -16315,6 +17646,7 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ ** during a hash table resize is a benign fault. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_BUILTIN_TEST @@ -16396,6 +17728,7 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ ** are merely placeholders. Real drivers must be substituted using ** sqlite3_config() before SQLite will operate. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -16482,6 +17815,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** be necessary when compiling for Delphi, ** for example. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -16519,9 +17853,9 @@ static malloc_zone_t* _sqliteZone_; ** The malloc.h header file is needed for malloc_usable_size() function ** on some systems (e.g. Linux). */ -#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE) -# define SQLITE_USE_MALLOC_H -# define SQLITE_USE_MALLOC_USABLE_SIZE +#if HAVE_MALLOC_H && HAVE_MALLOC_USABLE_SIZE +# define SQLITE_USE_MALLOC_H 1 +# define SQLITE_USE_MALLOC_USABLE_SIZE 1 /* ** The MSVCRT has malloc_usable_size(), but it is called _msize(). The ** use of _msize() is automatic, but can be disabled by compiling with @@ -16757,6 +18091,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only if the @@ -17291,6 +18626,7 @@ SQLITE_PRIVATE int sqlite3MemdebugMallocCount(){ ** This version of the memory allocation subsystem is included ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is only built into the library @@ -18005,6 +19341,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** The sqlite3_status() logic tracks the maximum values of n and M so ** that an application can, at any time, verify this constraint. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only when @@ -18548,6 +19885,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** ** This file contains code that is common across all mutex implementations. */ +/* #include "sqliteInt.h" */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) /* @@ -18556,7 +19894,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -18579,11 +19917,18 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ }else{ pFrom = sqlite3NoopMutex(); } - memcpy(pTo, pFrom, offsetof(sqlite3_mutex_methods, xMutexAlloc)); - memcpy(&pTo->xMutexFree, &pFrom->xMutexFree, - sizeof(*pTo) - offsetof(sqlite3_mutex_methods, xMutexFree)); + pTo->xMutexInit = pFrom->xMutexInit; + pTo->xMutexEnd = pFrom->xMutexEnd; + pTo->xMutexFree = pFrom->xMutexFree; + pTo->xMutexEnter = pFrom->xMutexEnter; + pTo->xMutexTry = pFrom->xMutexTry; + pTo->xMutexLeave = pFrom->xMutexLeave; + pTo->xMutexHeld = pFrom->xMutexHeld; + pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -18613,10 +19958,12 @@ SQLITE_PRIVATE int sqlite3MutexEnd(void){ /* ** Retrieve a pointer to a static mutex or allocate a new dynamic one. */ -SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -18625,14 +19972,16 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } /* ** Free a dynamic mutex. */ -SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -18641,8 +19990,9 @@ SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ ** Obtain the mutex p. If some other thread already has the mutex, block ** until it can be obtained. */ -SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -18651,9 +20001,10 @@ SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ ** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another ** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. */ -SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -18665,8 +20016,9 @@ SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ ** is not currently entered. If a NULL pointer is passed as an argument ** this function is a no-op. */ -SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -18676,10 +20028,12 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. */ -SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } -SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif @@ -18715,6 +20069,7 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ ** that does error checking on mutexes to make sure they are being ** called correctly. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_MUTEX_OMIT @@ -18796,7 +20151,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: @@ -18809,8 +20164,12 @@ static sqlite3_mutex *debugMutexAlloc(int id){ break; } default: { - assert( id-2 >= 0 ); - assert( id-2 < (int)(sizeof(aStatic)/sizeof(aStatic[0])) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( id-2<0 || id-2>=ArraySize(aStatic) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif pNew = &aStatic[id-2]; pNew->id = id; break; @@ -18825,8 +20184,13 @@ static sqlite3_mutex *debugMutexAlloc(int id){ static void debugMutexFree(sqlite3_mutex *pX){ sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; assert( p->cnt==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); - sqlite3_free(p); + if( p->id==SQLITE_MUTEX_RECURSIVE || p->id==SQLITE_MUTEX_FAST ){ + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; +#endif + } } /* @@ -18909,6 +20273,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for pthreads */ +/* #include "sqliteInt.h" */ /* ** The code in this file is only used if we are compiling threadsafe @@ -18937,8 +20302,10 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ */ struct sqlite3_mutex { pthread_mutex_t mutex; /* Mutex controlling the lock */ -#if SQLITE_MUTEX_NREF +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) int id; /* Mutex type */ +#endif +#if SQLITE_MUTEX_NREF volatile int nRef; /* Number of entrances */ volatile pthread_t owner; /* Thread that is within this mutex */ int trace; /* True to trace changes */ @@ -18975,6 +20342,19 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ @@ -19000,6 +20380,9 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -19036,6 +20419,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; @@ -19054,9 +20440,6 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&p->mutex, &recursiveAttr); pthread_mutexattr_destroy(&recursiveAttr); -#endif -#if SQLITE_MUTEX_NREF - p->id = iType; #endif } break; @@ -19064,23 +20447,24 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: { p = sqlite3MallocZero( sizeof(*p) ); if( p ){ -#if SQLITE_MUTEX_NREF - p->id = iType; -#endif pthread_mutex_init(&p->mutex, 0); } break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); - p = &staticMutexes[iType-2]; -#if SQLITE_MUTEX_NREF - p->id = iType; +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + p = &staticMutexes[iType-2]; break; } } +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + if( p ) p->id = iType; +#endif return p; } @@ -19092,9 +20476,18 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); - pthread_mutex_destroy(&p->mutex); - sqlite3_free(p); +#if SQLITE_ENABLE_API_ARMOR + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) +#endif + { + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif } /* @@ -19268,6 +20661,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for Win32. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* @@ -19306,16 +20700,6 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. @@ -19564,6 +20948,17 @@ SQLITE_API int sqlite3_open_file_count = 0; # define SQLITE_WIN32_VOLATILE volatile #endif +/* +** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() +** functions are not available (e.g. those not using MSVC, Cygwin, etc). +*/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) +# define SQLITE_OS_WIN_THREADS 1 +#else +# define SQLITE_OS_WIN_THREADS 0 +#endif + #endif /* _OS_WIN_H_ */ /************** End of os_win.h **********************************************/ @@ -19622,6 +21017,24 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ + defined(_MSC_VER) && _MSC_VER>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) + MemoryBarrier(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ @@ -19634,6 +21047,9 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; @@ -19646,8 +21062,8 @@ static int winMutex_isNt = -1; /* <0 means "need to query" */ */ static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; -SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ -SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void); /* os_win.c */ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ @@ -19705,6 +21121,9 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -19739,8 +21158,8 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); if( p ){ -#ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC p->trace = 1; #endif @@ -19754,12 +21173,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(winMutex_staticMutexes) ); - assert( winMutex_isInit==1 ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(winMutex_staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &winMutex_staticMutexes[iType-2]; -#ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC p->trace = 1; #endif @@ -19778,13 +21200,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); -#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); - assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ){ + DeleteCriticalSection(&p->mutex); + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; #endif - assert( winMutex_isInit==1 ); - DeleteCriticalSection(&p->mutex); - sqlite3_free(p); + } } /* @@ -19931,6 +21355,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** ** Memory allocation functions used throughout sqlite. */ +/* #include "sqliteInt.h" */ /* #include */ /* @@ -19938,7 +21363,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** held by SQLite. An example of non-essential memory is memory used to ** cache database pages that are not currently in use. */ -SQLITE_API int sqlite3_release_memory(int n){ +SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int n){ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT return sqlite3PcacheReleaseMemory(n); #else @@ -19963,16 +21388,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -19989,54 +21405,32 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) /* -** This routine runs when the memory allocator sees that the -** total memory allocation is about to exceed the soft heap -** limit. +** Return the memory allocator mutex. sqlite3_status() needs it. */ -static void softHeapLimitEnforcer( - void *NotUsed, - sqlite3_int64 NotUsed2, - int allocSize -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_release_memory(allocSize); -} - -/* -** Change the alarm callback -*/ -static int sqlite3MemoryAlarm( - void(*xCallback)(void *pArg, sqlite3_int64 used,int N), - void *pArg, - sqlite3_int64 iThreshold -){ - int nUsed; - sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; - mem0.alarmThreshold = iThreshold; - nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); - sqlite3_mutex_leave(mem0.mutex); - return SQLITE_OK; +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ + return mem0.mutex; } #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite3MemoryAlarm. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ -SQLITE_API int sqlite3_memory_alarm( +SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ - return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); + (void)xCallback; + (void)pArg; + (void)iThreshold; + return SQLITE_OK; } #endif @@ -20044,27 +21438,29 @@ SQLITE_API int sqlite3_memory_alarm( ** Set the soft heap-size limit for the library. Passing a zero or ** negative value indicates no limit. */ -SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - if( n<0 ) return priorLimit; - if( n>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); - }else{ - sqlite3MemoryAlarm(0, 0, 0); + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; } -SQLITE_API void sqlite3_soft_heap_limit(int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_soft_heap_limit(int n){ if( n<0 ) n = 0; sqlite3_soft_heap_limit64(n); } @@ -20073,6 +21469,7 @@ SQLITE_API void sqlite3_soft_heap_limit(int n){ ** Initialize the memory allocation subsystem. */ SQLITE_PRIVATE int sqlite3MallocInit(void){ + int rc; if( sqlite3GlobalConfig.m.xMalloc==0 ){ sqlite3MemSetDefault(); } @@ -20103,12 +21500,13 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3GlobalConfig.nScratch = 0; } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 - || sqlite3GlobalConfig.nPage<1 ){ + || sqlite3GlobalConfig.nPage<=0 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; - sqlite3GlobalConfig.nPage = 0; } - return sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); + rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); + if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0)); + return rc; } /* @@ -20133,11 +21531,9 @@ SQLITE_PRIVATE void sqlite3MallocEnd(void){ /* ** Return the amount of memory currently checked out. */ -SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); - res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); return res; } @@ -20146,31 +21542,20 @@ SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ ** checked out since either the beginning of this process ** or since the most recent reset. */ -SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); - res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ - return res; +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag){ + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); + return mx; } /* ** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ - void (*xCallback)(void*,sqlite3_int64,int); - sqlite3_int64 nowUsed; - void *pArg; - if( mem0.alarmCallback==0 ) return; - xCallback = mem0.alarmCallback; - nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - pArg = mem0.alarmArg; - mem0.alarmCallback = 0; + if( mem0.alarmThreshold<=0 ) return; sqlite3_mutex_leave(mem0.mutex); - xCallback(pArg, nowUsed, nByte); + sqlite3_release_memory(nByte); sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; } /* @@ -20183,8 +21568,8 @@ static int mallocWithAlarm(int n, void **pp){ assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - if( mem0.alarmCallback!=0 ){ - int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -20194,15 +21579,15 @@ static int mallocWithAlarm(int n, void **pp){ } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmCallback ){ + if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } #endif if( p ){ nFull = sqlite3MallocSize(p); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); - sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; return nFull; @@ -20237,13 +21622,13 @@ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ ** First make sure the memory subsystem is initialized, then do the ** allocation. */ -SQLITE_API void *sqlite3_malloc(int n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif return n<=0 ? 0 : sqlite3Malloc(n); } -SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -20279,14 +21664,14 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); + sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1); sqlite3_mutex_leave(mem0.mutex); }else{ sqlite3_mutex_leave(mem0.mutex); p = sqlite3Malloc(n); if( sqlite3GlobalConfig.bMemstat && p ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); + sqlite3StatusUp(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); sqlite3_mutex_leave(mem0.mutex); } sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); @@ -20295,11 +21680,12 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocations per thread - ** are outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut<=1 ); + /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch + ** buffers per thread. + ** + ** This can only be checked in single-threaded mode. + */ + assert( scratchAllocOut==0 ); if( p ) scratchAllocOut++; #endif @@ -20326,19 +21712,19 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ mem0.pScratchFree = pSlot; mem0.nScratchFree++; assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch ); - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); + sqlite3StatusDown(SQLITE_STATUS_SCRATCH_USED, 1); sqlite3_mutex_leave(mem0.mutex); }else{ /* Release memory back to the heap */ assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); - assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_SCRATCH) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); if( sqlite3GlobalConfig.bMemstat ){ int iSize = sqlite3MallocSize(p); sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); - sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3StatusDown(SQLITE_STATUS_SCRATCH_OVERFLOW, iSize); + sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, iSize); + sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -20368,23 +21754,24 @@ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return sqlite3MallocSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; + if( db==0 || !isLookaside(db,p) ){ +#if SQLITE_DEBUG + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - return sqlite3GlobalConfig.m.xSize(p); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } +#endif + return sqlite3GlobalConfig.m.xSize(p); + }else{ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.sz; } } -SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ - assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); +SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); } @@ -20392,14 +21779,14 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ /* ** Free memory previously obtained from sqlite3Malloc(). */ -SQLITE_API void sqlite3_free(void *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); - sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, sqlite3MallocSize(p)); + sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -20440,7 +21827,7 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); @@ -20453,7 +21840,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, (u8)~MEMTYPE_HEAP) ); if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ } @@ -20481,13 +21868,13 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); - if( pNew==0 && mem0.alarmCallback ){ + if( pNew==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } if( pNew ){ nNew = sqlite3MallocSize(pNew); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nNew-nOld); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); } sqlite3_mutex_leave(mem0.mutex); }else{ @@ -20501,14 +21888,14 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ ** The public interface to sqlite3Realloc. Make sure that the memory ** subsystem is initialized prior to invoking sqliteRealloc. */ -SQLITE_API void *sqlite3_realloc(void *pOld, int n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void *pOld, int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif if( n<0 ) n = 0; /* IMP: R-26507-47431 */ return sqlite3Realloc(pOld, n); } -SQLITE_API void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -20620,7 +22007,7 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ } }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ @@ -20682,19 +22069,11 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ } /* -** Create a string from the zFromat argument and the va_list that follows. -** Store the string in memory obtained from sqliteMalloc() and make *pz -** point to that string. +** Free any prior content in *pz and replace it with a copy of zNew. */ -SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ - va_list ap; - char *z; - - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ sqlite3DbFree(db, *pz); - *pz = z; + *pz = sqlite3DbStrDup(db, zNew); } /* @@ -20715,17 +22094,16 @@ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ ** function. However, if a malloc() failure has occurred since the previous ** invocation SQLITE_NOMEM is returned instead. ** -** If the first argument, db, is not NULL and a malloc() error has occurred, -** then the connection error-code (the value returned by sqlite3_errcode()) -** is set to SQLITE_NOMEM. +** If an OOM as occurred, then the connection error-code (the value +** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ - /* If the db handle is not NULL, then we must hold the connection handle - ** mutex here. Otherwise the read (and possible write) of db->mallocFailed + /* If the db handle must hold the connection handle mutex here. + ** Otherwise the read (and possible write) of db->mallocFailed ** is unsafe, as is the call to sqlite3Error(). */ - assert( !db || sqlite3_mutex_held(db->mutex) ); - if( db==0 ) return rc & 0xff; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ return apiOomError(db); } @@ -20736,29 +22114,16 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ /************** Begin file printf.c ******************************************/ /* ** The "printf" code that follows dates from the 1980's. It is in -** the public domain. The original comments are included here for -** completeness. They are very out-of-date but might be useful as -** an historical reference. Most of the "enhancements" have been backed -** out so that the functionality is now the same as standard printf(). +** the public domain. ** ************************************************************************** ** ** This file contains code for a set of "printf"-like routines. These ** routines format strings much like the printf() from the standard C ** library, though the implementation here has enhancements to support -** SQLlite. -*/ - -/* -** If the strchrnul() library function is available, then set -** HAVE_STRCHRNUL. If that routine is not available, this module -** will supply its own. The built-in version is slower than -** the glibc version so the glibc version is definitely preferred. +** SQLite. */ -#if !defined(HAVE_STRCHRNUL) -# define HAVE_STRCHRNUL 0 -#endif - +/* #include "sqliteInt.h" */ /* ** Conversion types fall into various categories as defined by the @@ -20884,6 +22249,7 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ ** Set the StrAccum object to an error mode. */ static void setStrAccumError(StrAccum *p, u8 eError){ + assert( eError==STRACCUM_NOMEM || eError==STRACCUM_TOOBIG ); p->accError = eError; p->nAlloc = 0; } @@ -20998,7 +22364,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } }while( !done && (c=(*++fmt))!=0 ); /* Get the field width */ - width = 0; if( c=='*' ){ if( bArgList ){ width = (int)getIntArg(pArgList); @@ -21007,18 +22372,21 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } if( width<0 ){ flag_leftjustify = 1; - width = -width; + width = width >= -2147483647 ? -width : 0; } c = *++fmt; }else{ + unsigned wx = 0; while( c>='0' && c<='9' ){ - width = width*10 + c - '0'; + wx = wx*10 + c - '0'; c = *++fmt; } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; } + /* Get the precision */ if( c=='.' ){ - precision = 0; c = *++fmt; if( c=='*' ){ if( bArgList ){ @@ -21026,13 +22394,18 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ precision = va_arg(ap,int); } - if( precision<0 ) precision = -precision; c = *++fmt; + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } }else{ + unsigned px = 0; while( c>='0' && c<='9' ){ - precision = precision*10 + c - '0'; + px = px*10 + c - '0'; c = *++fmt; } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; } }else{ precision = -1; @@ -21196,7 +22569,8 @@ SQLITE_PRIVATE void sqlite3VXPrintf( else prefix = 0; } if( xtype==etGENERIC && precision>0 ) precision--; - for(idx=precision, rounder=0.5; idx>0; idx--, rounder*=0.1){} + testcase( precision>0xfff ); + for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} if( xtype==etFLOAT ) realvalue += rounder; /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; @@ -21208,21 +22582,16 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( realvalue>0.0 ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } - while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } realvalue /= scale; while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - if( prefix=='-' ){ - bufpt = "-Inf"; - }else if( prefix=='+' ){ - bufpt = "+Inf"; - }else{ - bufpt = "Inf"; - } - length = sqlite3Strlen30(bufpt); + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); break; } } @@ -21251,8 +22620,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ e2 = exp; } - if( MAX(e2,0)+precision+width > etBUFSIZE - 15 ){ - bufpt = zExtra = sqlite3Malloc( MAX(e2,0)+precision+width+15 ); + if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ + bufpt = zExtra + = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); if( bufpt==0 ){ setStrAccumError(pAccum, STRACCUM_NOMEM); return; @@ -21370,12 +22740,13 @@ SQLITE_PRIVATE void sqlite3VXPrintf( case etDYNSTRING: if( bArgList ){ bufpt = getTextArg(pArgList); + xtype = etSTRING; }else{ bufpt = va_arg(ap,char*); } if( bufpt==0 ){ bufpt = ""; - }else if( xtype==etDYNSTRING && !bArgList ){ + }else if( xtype==etDYNSTRING ){ zExtra = bufpt; } if( precision>=0 ){ @@ -21384,9 +22755,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = sqlite3Strlen30(bufpt); } break; - case etSQLESCAPE: - case etSQLESCAPE2: - case etSQLESCAPE3: { + case etSQLESCAPE: /* Escape ' characters */ + case etSQLESCAPE2: /* Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -21405,7 +22776,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; - n += i + 1 + needQuote*2; + n += i + 3; if( n>etBUFSIZE ){ bufpt = zExtra = sqlite3Malloc( n ); if( bufpt==0 ){ @@ -21484,13 +22855,13 @@ SQLITE_PRIVATE void sqlite3VXPrintf( */ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zNew; - assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==STRACCUM_TOOBIG); testcase(p->accError==STRACCUM_NOMEM); return 0; } - if( !p->useMalloc ){ + if( p->mxAlloc==0 ){ N = p->nAlloc - p->nChar - 1; setStrAccumError(p, STRACCUM_TOOBIG); return N; @@ -21498,6 +22869,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; szNew += N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } if( szNew > p->mxAlloc ){ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_TOOBIG); @@ -21505,15 +22881,16 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ }else{ p->nAlloc = (int)szNew; } - if( p->useMalloc==1 ){ + if( p->db ){ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); + zNew = sqlite3_realloc64(zOld, p->nAlloc); } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); @@ -21527,7 +22904,10 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ ** Append N copies of character c to the given string buffer. */ SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ - if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + testcase( p->nChar + (i64)N > 0x7fffffff ); + if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ + return; + } while( (N--)>0 ) p->zText[p->nChar++] = c; } @@ -21552,7 +22932,7 @@ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ ** size of the memory allocation for StrAccum if necessary. */ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ - assert( z!=0 ); + assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); assert( p->accError==0 || p->nAlloc==0 ); @@ -21581,12 +22961,8 @@ SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ p->zText[p->nChar] = 0; - if( p->useMalloc && p->zText==p->zBase ){ - if( p->useMalloc==1 ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - }else{ - p->zText = sqlite3_malloc(p->nChar+1); - } + if( p->mxAlloc>0 && p->zText==p->zBase ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); }else{ @@ -21602,25 +22978,31 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ if( p->zText!=p->zBase ){ - if( p->useMalloc==1 ){ - sqlite3DbFree(p->db, p->zText); - }else{ - sqlite3_free(p->zText); - } + sqlite3DbFree(p->db, p->zText); } p->zText = 0; } /* -** Initialize a string accumulator +** Initialize a string accumulator. +** +** p: The accumulator to be initialized. +** db: Pointer to a database connection. May be NULL. Lookaside +** memory is used if not NULL. db->mallocFailed is set appropriately +** when not NULL. +** zBase: An initial buffer. May be NULL in which case the initial buffer +** is malloced. +** n: Size of zBase in bytes. If total space requirements never exceed +** n then no memory allocations ever occur. +** mx: Maximum number of bytes to accumulate. If mx==0 then no memory +** allocations will ever occur. */ -SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n, int mx){ +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ p->zText = p->zBase = zBase; - p->db = 0; + p->db = db; p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; - p->useMalloc = 1; p->accError = 0; } @@ -21633,9 +23015,8 @@ SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list a char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; assert( db!=0 ); - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), + sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - acc.db = db; sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ @@ -21657,37 +23038,25 @@ SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ return z; } -/* -** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting -** the string and before returning. This routine is intended to be used -** to modify an existing string. For example: -** -** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); -** -*/ -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - sqlite3DbFree(db, zStr); - return z; -} - /* ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ +SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - acc.useMalloc = 2; + sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; @@ -21697,7 +23066,7 @@ SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ ** Print into memory obtained from sqlite3_malloc()(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ +SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char *zFormat, ...){ va_list ap; char *z; #ifndef SQLITE_OMIT_AUTOINIT @@ -21722,15 +23091,21 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ ** ** sqlite3_vsnprintf() is the varargs version. */ -SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ +SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; - sqlite3StrAccumInit(&acc, zBuf, n, 0); - acc.useMalloc = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); sqlite3VXPrintf(&acc, 0, zFormat, ap); return sqlite3StrAccumFinish(&acc); } -SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ +SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ char *z; va_list ap; va_start(ap,zFormat); @@ -21747,13 +23122,17 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ ** sqlite3_log() must render into a static buffer. It cannot dynamically ** allocate memory because it might be called while the memory allocator ** mutex is held. +** +** sqlite3VXPrintf() might ask for *temporary* memory allocations for +** certain format characters (%q) or for very large precisions or widths. +** Care must be taken that any sqlite3_log() calls that occur while the +** memory mutex is held do not use these mechanisms. */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ - sqlite3StrAccumInit(&acc, zMsg, sizeof(zMsg), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3VXPrintf(&acc, 0, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); @@ -21762,7 +23141,7 @@ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ /* ** Format and write a message to the log if logging is enabled. */ -SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ +SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...){ va_list ap; /* Vararg list */ if( sqlite3GlobalConfig.xLog ){ va_start(ap, zFormat); @@ -21771,7 +23150,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ } } -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) /* ** A version of printf() that understands %lld. Used for debugging. ** The printf() built into some versions of windows does not understand %lld @@ -21781,8 +23160,7 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ va_list ap; StrAccum acc; char zBuf[500]; - sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); @@ -21792,24 +23170,49 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif -#ifdef SQLITE_DEBUG -/************************************************************************* -** Routines for implementing the "TreeView" display of hierarchical -** data structures for debugging. + +/* +** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. +*/ +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3VXPrintf(p, bFlags, zFormat, ap); + va_end(ap); +} + +/************** End of printf.c **********************************************/ +/************** Begin file treeview.c ****************************************/ +/* +** 2015-06-08 ** -** The main entry points (coded elsewhere) are: -** sqlite3TreeViewExpr(0, pExpr, 0); -** sqlite3TreeViewExprList(0, pList, 0, 0); -** sqlite3TreeViewSelect(0, pSelect, 0); -** Insert calls to those routines while debugging in order to display -** a diagram of Expr, ExprList, and Select objects. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains C code to implement the TreeView debugging routines. +** These routines print a parse tree to standard output for debugging and +** analysis. +** +** The interfaces in this file is only available when compiling +** with SQLITE_DEBUG. */ -/* Add a new subitem to the tree. The moreToFollow flag indicates that this -** is not the last item in the tree. */ -SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ +/* #include "sqliteInt.h" */ +#ifdef SQLITE_DEBUG + +/* +** Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. +*/ +static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p==0 ){ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return 0; memset(p, 0, sizeof(*p)); }else{ @@ -21819,21 +23222,26 @@ SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; return p; } -/* Finished with one layer of the tree */ -SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView *p){ + +/* +** Finished with one layer of the tree +*/ +static void sqlite3TreeViewPop(TreeView *p){ if( p==0 ) return; p->iLevel--; if( p->iLevel<0 ) sqlite3_free(p); } -/* Generate a single line of output for the tree, with a prefix that contains -** all the appropriate tree lines */ -SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + +/* +** Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines +*/ +static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; char zBuf[500]; - sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); if( p ){ for(i=0; iiLevel && ibLine)-1; i++){ sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); @@ -21848,24 +23256,378 @@ SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ fprintf(stdout,"%s", zBuf); fflush(stdout); } -/* Shorthand for starting a new tree item that consists of a single label */ -SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView *p, const char *zLabel, u8 moreToFollow){ - p = sqlite3TreeViewPush(p, moreToFollow); + +/* +** Shorthand for starting a new tree item that consists of a single label +*/ +static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ + p = sqlite3TreeViewPush(p, moreFollows); sqlite3TreeViewLine(p, "%s", zLabel); } -#endif /* SQLITE_DEBUG */ + /* -** variable-argument wrapper around sqlite3VXPrintf(). +** Generate a human-readable description of a the Select object. */ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ - va_list ap; - va_start(ap,zFormat); - sqlite3VXPrintf(p, bFlags, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + int cnt = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + do{ + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags + ); + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + } + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); + sqlite3TreeViewPop(pView); } -/************** End of printf.c **********************************************/ +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + char zFlgs[30]; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } + if( pExpr->flags ){ + sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + }else{ + zFlgs[0] = 0; + } + switch( pExpr->op ){ + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_COLLATE: { + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, 0, 0); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s%s", zBinOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} + +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + int i; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + if( j ){ + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + } + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); + if( j ) sqlite3TreeViewPop(pView); + } + } + sqlite3TreeViewPop(pView); +} + +#endif /* SQLITE_DEBUG */ + +/************** End of treeview.c ********************************************/ /************** Begin file random.c ******************************************/ /* ** 2001 September 15 @@ -21884,6 +23646,7 @@ SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ** Random numbers are used by some of the database backends in order ** to generate random integer keys for tables or random filenames. */ +/* #include "sqliteInt.h" */ /* All threads share a single random number generator. @@ -21898,7 +23661,7 @@ static SQLITE_WSD struct sqlite3PrngType { /* ** Return N random bytes. */ -SQLITE_API void sqlite3_randomness(int N, void *pBuf){ +SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; @@ -21916,11 +23679,19 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; #endif - if( N<=0 ){ +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + + sqlite3_mutex_enter(mutex); + if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; sqlite3_mutex_leave(mutex); return; @@ -22022,6 +23793,10 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ ** of multiple cores can do so, while also allowing applications to stay ** single-threaded if desired. */ +/* #include "sqliteInt.h" */ +#if SQLITE_OS_WIN +/* # include "os_win.h" */ +#endif #if SQLITE_MAX_WORKER_THREADS>0 @@ -22060,6 +23835,10 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; }else{ @@ -22094,7 +23873,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /********************************* Win32 Threads ****************************/ -#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 +#if SQLITE_OS_WIN_THREADS #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include @@ -22144,7 +23923,12 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; - if( sqlite3GlobalConfig.bCoreMutex==0 ){ + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ memset(p, 0, sizeof(*p)); }else{ p->xTask = xTask; @@ -22172,7 +23956,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ - assert( p->id==GetCurrentThreadId() ); + /* assert( p->id==GetCurrentThreadId() ); */ rc = WAIT_OBJECT_0; assert( p->tid==0 ); }else{ @@ -22187,7 +23971,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; } -#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT */ +#endif /* SQLITE_OS_WIN_THREADS */ /******************************** End Win32 Threads *************************/ @@ -22294,7 +24078,9 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ ** 0xfe 0xff big-endian utf-16 follows ** */ +/* #include "sqliteInt.h" */ /* #include */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_AMALGAMATION /* @@ -22807,8 +24593,9 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ ** strings, and stuff like that. ** */ +/* #include "sqliteInt.h" */ /* #include */ -#ifdef SQLITE_HAVE_ISNAN +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN # include #endif @@ -22849,7 +24636,7 @@ SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ */ SQLITE_PRIVATE int sqlite3IsNaN(double x){ int rc; /* The value return */ -#if !defined(SQLITE_HAVE_ISNAN) +#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN /* ** Systems that support the isnan() library function should probably ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have @@ -22879,9 +24666,9 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ volatile double y = x; volatile double z = y; rc = (y!=z); -#else /* if defined(SQLITE_HAVE_ISNAN) */ +#else /* if HAVE_ISNAN */ rc = isnan(x); -#endif /* SQLITE_HAVE_ISNAN */ +#endif /* HAVE_ISNAN */ testcase( rc ); return rc; } @@ -22896,10 +24683,8 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ ** than 1GiB) the value returned might be less than the true string length. */ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ - const char *z2 = z; if( z==0 ) return 0; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); + return 0x3fffffff & (int)strlen(z); } /* @@ -23040,15 +24825,25 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ ** case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ +SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } return UpperToLower[*a] - UpperToLower[*b]; } -SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ +SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -23436,6 +25231,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ } } #endif + while( zNum[0]=='0' ) zNum++; for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ v = v*10 + c; } @@ -23847,11 +25643,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ ** 64-bit integer. */ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 && ALWAYS(i<9) ); + int i; + for(i=1; (v >>= 7)!=0; i++){ assert( i<9 ); } return i; } @@ -23860,14 +25653,40 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ ** Read or write a four-byte big-endian integer value. */ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ +#if SQLITE_BYTEORDER==4321 + u32 x; + memcpy(&x,p,4); + return x; +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x; + memcpy(&x,p,4); + return __builtin_bswap32(x); +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x; + memcpy(&x,p,4); + return _byteswap_ulong(x); +#else testcase( p[0]&0x80 ); return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +#endif } SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ +#if SQLITE_BYTEORDER==4321 + memcpy(p,&v,4); +#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x = __builtin_bswap32(v); + memcpy(p,&x,4); +#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x = _byteswap_ulong(v); + memcpy(p,&x,4); +#else p[0] = (u8)(v>>24); p[1] = (u8)(v>>16); p[2] = (u8)(v>>8); p[3] = (u8)v; +#endif } @@ -24170,6 +25989,7 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ ** This is the implementation of generic hash-tables ** used in SQLite. */ +/* #include "sqliteInt.h" */ /* #include */ /* Turn bulk memory into a hash table object by initializing the @@ -24435,42 +26255,42 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ #endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ static const char *const azName[] = { "?", - /* 1 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 2 */ "Savepoint" OpHelp(""), - /* 3 */ "AutoCommit" OpHelp(""), - /* 4 */ "Transaction" OpHelp(""), - /* 5 */ "SorterNext" OpHelp(""), - /* 6 */ "PrevIfOpen" OpHelp(""), - /* 7 */ "NextIfOpen" OpHelp(""), - /* 8 */ "Prev" OpHelp(""), - /* 9 */ "Next" OpHelp(""), - /* 10 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 11 */ "Checkpoint" OpHelp(""), - /* 12 */ "JournalMode" OpHelp(""), - /* 13 */ "Vacuum" OpHelp(""), - /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 16 */ "Goto" OpHelp(""), - /* 17 */ "Gosub" OpHelp(""), - /* 18 */ "Return" OpHelp(""), + /* 1 */ "Savepoint" OpHelp(""), + /* 2 */ "AutoCommit" OpHelp(""), + /* 3 */ "Transaction" OpHelp(""), + /* 4 */ "SorterNext" OpHelp(""), + /* 5 */ "PrevIfOpen" OpHelp(""), + /* 6 */ "NextIfOpen" OpHelp(""), + /* 7 */ "Prev" OpHelp(""), + /* 8 */ "Next" OpHelp(""), + /* 9 */ "Checkpoint" OpHelp(""), + /* 10 */ "JournalMode" OpHelp(""), + /* 11 */ "Vacuum" OpHelp(""), + /* 12 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 13 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 14 */ "Goto" OpHelp(""), + /* 15 */ "Gosub" OpHelp(""), + /* 16 */ "Return" OpHelp(""), + /* 17 */ "InitCoroutine" OpHelp(""), + /* 18 */ "EndCoroutine" OpHelp(""), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "InitCoroutine" OpHelp(""), - /* 21 */ "EndCoroutine" OpHelp(""), - /* 22 */ "Yield" OpHelp(""), - /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 24 */ "Halt" OpHelp(""), - /* 25 */ "Integer" OpHelp("r[P2]=P1"), - /* 26 */ "Int64" OpHelp("r[P2]=P4"), - /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 36 */ "CollSeq" OpHelp(""), + /* 20 */ "Yield" OpHelp(""), + /* 21 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 22 */ "Halt" OpHelp(""), + /* 23 */ "Integer" OpHelp("r[P2]=P1"), + /* 24 */ "Int64" OpHelp("r[P2]=P4"), + /* 25 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 26 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 27 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 28 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 29 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 30 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 31 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 32 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 33 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 34 */ "CollSeq" OpHelp(""), + /* 35 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), + /* 36 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), /* 38 */ "MustBeInt" OpHelp(""), /* 39 */ "RealAffinity" OpHelp(""), @@ -24496,20 +26316,20 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 59 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), /* 60 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), /* 61 */ "Close" OpHelp(""), - /* 62 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 63 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 64 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 65 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 66 */ "Seek" OpHelp("intkey=r[P2]"), - /* 67 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 68 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 69 */ "Found" OpHelp("key=r[P3@P4]"), - /* 70 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 62 */ "ColumnsUsed" OpHelp(""), + /* 63 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 64 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 65 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 66 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 67 */ "Seek" OpHelp("intkey=r[P2]"), + /* 68 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 69 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 70 */ "Found" OpHelp("key=r[P3@P4]"), /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 73 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 74 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 75 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 73 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 74 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 75 */ "NewRowid" OpHelp("r[P2]=rowid"), /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), @@ -24518,7 +26338,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), - /* 84 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 84 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 goto P2"), - /* 138 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), - /* 139 */ "IfZero" OpHelp("r[P1]+=P3, if r[P1]==0 goto P2"), - /* 140 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 141 */ "IncrVacuum" OpHelp(""), - /* 142 */ "Expire" OpHelp(""), - /* 143 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 144 */ "VBegin" OpHelp(""), - /* 145 */ "VCreate" OpHelp(""), - /* 146 */ "VDestroy" OpHelp(""), - /* 147 */ "VOpen" OpHelp(""), - /* 148 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 149 */ "VNext" OpHelp(""), - /* 150 */ "VRename" OpHelp(""), - /* 151 */ "Pagecount" OpHelp(""), - /* 152 */ "MaxPgcnt" OpHelp(""), - /* 153 */ "Init" OpHelp("Start at P2"), - /* 154 */ "Noop" OpHelp(""), - /* 155 */ "Explain" OpHelp(""), + /* 134 */ "Param" OpHelp(""), + /* 135 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 136 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 137 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 138 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 139 */ "SetIfNotPos" OpHelp("if r[P1]<=0 then r[P2]=P3"), + /* 140 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), + /* 141 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 142 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), + /* 143 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 144 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 145 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 146 */ "IncrVacuum" OpHelp(""), + /* 147 */ "Expire" OpHelp(""), + /* 148 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 149 */ "VBegin" OpHelp(""), + /* 150 */ "VCreate" OpHelp(""), + /* 151 */ "VDestroy" OpHelp(""), + /* 152 */ "VOpen" OpHelp(""), + /* 153 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 154 */ "VNext" OpHelp(""), + /* 155 */ "VRename" OpHelp(""), + /* 156 */ "Pagecount" OpHelp(""), + /* 157 */ "MaxPgcnt" OpHelp(""), + /* 158 */ "Init" OpHelp("Start at P2"), + /* 159 */ "Noop" OpHelp(""), + /* 160 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -24642,6 +26467,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_UNIX /* This file is used on unix only */ /* @@ -24669,18 +26495,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # endif #endif -/* -** Define the OS_VXWORKS pre-processor macro to 1 if building on -** vxworks, or 0 otherwise. -*/ -#ifndef OS_VXWORKS -# if defined(__RTP__) || defined(_WRS_KERNEL) -# define OS_VXWORKS 1 -# else -# define OS_VXWORKS 0 -# endif -#endif - /* ** standard include files. */ @@ -24695,18 +26509,30 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # include #endif -#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE # include -# if OS_VXWORKS -# include -# include -# else -# include -# include -# endif +# include +# include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif +#endif + + +#if OS_VXWORKS +/* # include */ +# include +# include +#endif /* OS_VXWORKS */ + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE # include #endif @@ -24747,6 +26573,10 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define MAX_PATHNAME 512 +/* Always cast the getpid() return type for compatibility with +** kernel modules in VxWorks. */ +#define osGetpid(X) (pid_t)getpid() + /* ** Only set the lastErrno if the error code is a real error and not ** a normal expected return code of SQLITE_BUSY or SQLITE_OK @@ -24835,7 +26665,7 @@ struct unixFile { ** method was called. If xOpen() is called from a different process id, ** indicating that a fork() has occurred, the PRNG will be reset. */ -static int randomnessPid = 0; +static pid_t randomnessPid = 0; /* ** Allowed values for the unixFile.ctrlFlags bitmask: @@ -24852,7 +26682,8 @@ static int randomnessPid = 0; #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ -#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings issued */ +#define UNIXFILE_BLOCK 0x0200 /* Next SHM lock might block */ /* ** Include code that is common to all os_*.c files @@ -24890,16 +26721,6 @@ static int randomnessPid = 0; # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. @@ -25191,7 +27012,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -25208,7 +27029,7 @@ static struct unix_syscall { { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -25430,19 +27251,19 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterLeave() */ static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +#ifdef SQLITE_HAVE_OS_TRACE /* ** Helper function for printing out trace information from debugging ** binaries. This returns the string representation of the supplied @@ -25523,9 +27344,9 @@ static int lockTrace(int fd, int op, struct flock *p){ /* ** Retry ftruncate() calls that fail due to EINTR ** -** All calls to ftruncate() within this file should be made through this wrapper. -** On the Android platform, bypassing the logic below could lead to a corrupt -** database. +** All calls to ftruncate() within this file should be made through +** this wrapper. On the Android platform, bypassing the logic below +** could lead to a corrupt database. */ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; @@ -25705,7 +27526,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ assert( zAbsoluteName[0]=='/' ); n = (int)strlen(zAbsoluteName); - pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); if( pNew==0 ) return 0; pNew->zCanonicalName = (char*)&pNew[1]; memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); @@ -25984,6 +27805,14 @@ static void robust_close(unixFile *pFile, int h, int lineno){ } } +/* +** Set the pFile->lastErrno. Do this in a subroutine as that provides +** a convenient place to set a breakpoint. +*/ +static void storeLastErrno(unixFile *pFile, int error){ + pFile->lastErrno = error; +} + /* ** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. */ @@ -26057,7 +27886,7 @@ static int findInodeInfo( fd = pFile->h; rc = osFstat(fd, &statbuf); if( rc!=0 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); #ifdef EOVERFLOW if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; #endif @@ -26078,12 +27907,12 @@ static int findInodeInfo( if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); if( rc!=1 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR; } rc = osFstat(fd, &statbuf); if( rc!=0 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR; } } @@ -26101,7 +27930,7 @@ static int findInodeInfo( pInode = pInode->pNext; } if( pInode==0 ){ - pInode = sqlite3_malloc( sizeof(*pInode) ); + pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ return SQLITE_NOMEM; } @@ -26206,7 +28035,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ lock.l_type = F_WRLCK; if( osFcntl(pFile->h, F_GETLK, &lock) ){ rc = SQLITE_IOERR_CHECKRESERVEDLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); } else if( lock.l_type!=F_UNLCK ){ reserved = 1; } @@ -26339,7 +28168,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared , getpid())); + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared, + osGetpid(0))); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the end_lock: exit path, as @@ -26406,7 +28236,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_lock; } @@ -26441,7 +28271,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( rc ){ if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_lock; }else{ @@ -26474,7 +28304,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( rc!=SQLITE_BUSY ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } } } @@ -26547,7 +28377,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); + osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); if( pFile->eFileLock<=eFileLock ){ @@ -26581,7 +28411,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** 4: [RRRR.] */ if( eFileLock==SHARED_LOCK ){ - #if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE (void)handleNFSUnlock; assert( handleNFSUnlock==0 ); @@ -26599,7 +28428,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_unlock; } @@ -26611,7 +28440,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_unlock; } @@ -26623,7 +28452,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } goto end_unlock; } @@ -26642,7 +28471,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** SQLITE_BUSY would confuse the upper layer (in practice it causes ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); goto end_unlock; } } @@ -26655,7 +28484,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ pInode->eFileLock = SHARED_LOCK; }else{ rc = SQLITE_IOERR_UNLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); goto end_unlock; } } @@ -26673,7 +28502,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ pInode->eFileLock = NO_LOCK; }else{ rc = SQLITE_IOERR_UNLOCK; - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); pInode->eFileLock = NO_LOCK; pFile->eFileLock = NO_LOCK; } @@ -26948,7 +28777,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { } else { rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } } return rc; @@ -26975,7 +28804,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -27002,7 +28831,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { rc = SQLITE_IOERR_UNLOCK; } if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } @@ -27038,10 +28867,9 @@ static int dotlockClose(sqlite3_file *id) { ** still works when you do this, but concurrency is reduced since ** only a single process can be reading the database at a time. ** -** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if -** compiling for VXWORKS. +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off */ -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE /* ** Retry flock() calls that fail with EINTR @@ -27089,7 +28917,7 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ /* unlock failed with an error */ lrc = SQLITE_IOERR_UNLOCK; if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); rc = lrc; } } @@ -27099,7 +28927,7 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ /* someone else might have it reserved */ lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); rc = lrc; } } @@ -27165,7 +28993,7 @@ static int flockLock(sqlite3_file *id, int eFileLock) { /* didn't get, must be busy */ rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } } else { /* got it, set the type and return ok */ @@ -27194,7 +29022,7 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -27255,7 +29083,7 @@ static int flockClose(sqlite3_file *id) { ** to a non-zero value otherwise *pResOut is set to zero. The return value ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { +static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; @@ -27277,7 +29105,7 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { int tErrno = errno; if( EAGAIN != tErrno ){ rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } else { /* someone else has the lock when we are in NO_LOCK */ reserved = (pFile->eFileLock < SHARED_LOCK); @@ -27322,7 +29150,7 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { ** This routine will only increase a lock. Use the sqlite3OsUnlock() ** routine to lower a locking level. */ -static int semLock(sqlite3_file *id, int eFileLock) { +static int semXLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; @@ -27355,14 +29183,14 @@ static int semLock(sqlite3_file *id, int eFileLock) { ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. */ -static int semUnlock(sqlite3_file *id, int eFileLock) { +static int semXUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; sem_t *pSem = pFile->pInode->pSem; assert( pFile ); assert( pSem ); OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ @@ -27381,7 +29209,7 @@ static int semUnlock(sqlite3_file *id, int eFileLock) { int rc, tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } @@ -27392,10 +29220,10 @@ static int semUnlock(sqlite3_file *id, int eFileLock) { /* ** Close a file. */ -static int semClose(sqlite3_file *id) { +static int semXClose(sqlite3_file *id) { if( id ){ unixFile *pFile = (unixFile*)id; - semUnlock(id, NO_LOCK); + semXUnlock(id, NO_LOCK); assert( pFile ); unixEnterMutex(); releaseInodeInfo(pFile); @@ -27483,7 +29311,7 @@ static int afpSetLock( setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); #endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + storeLastErrno(pFile, tErrno); } return rc; } else { @@ -27576,7 +29404,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + azFileLock(pInode->eFileLock), pInode->nShared , osGetpid(0))); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as @@ -27666,7 +29494,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); if( IS_LOCK_ERROR(lrc1) ) { - pFile->lastErrno = lrc1Errno; + storeLastErrno(pFile, lrc1Errno); rc = lrc1; goto afp_end_lock; } else if( IS_LOCK_ERROR(lrc2) ){ @@ -27762,7 +29590,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); + osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); if( pFile->eFileLock<=eFileLock ){ @@ -27940,7 +29768,6 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); - cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -27953,9 +29780,9 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ SimulateIOError( newOffset-- ); if( newOffset!=offset ){ if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; + storeLastErrno((unixFile*)id, errno); }else{ - ((unixFile*)id)->lastErrno = 0; + storeLastErrno((unixFile*)id, 0); } return -1; } @@ -27965,7 +29792,7 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ if( got<0 ){ if( errno==EINTR ){ got = 1; continue; } prior = 0; - ((unixFile*)id)->lastErrno = errno; + storeLastErrno((unixFile*)id, errno); break; }else if( got>0 ){ cnt -= got; @@ -28030,7 +29857,7 @@ static int unixRead( /* lastErrno set by seekAndRead */ return SQLITE_IOERR_READ; }else{ - pFile->lastErrno = 0; /* not a system error */ + storeLastErrno(pFile, 0); /* not a system error */ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[got], 0, amt-got); return SQLITE_IOERR_SHORT_READ; @@ -28059,9 +29886,9 @@ static int seekAndWriteFd( TIMER_START; #if defined(USE_PREAD) - do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); + do{ rc = (int)osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); #elif defined(USE_PREAD64) - do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); + do{ rc = (int)osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); #else do{ i64 iSeek = lseek(fd, iOff, SEEK_SET); @@ -28157,8 +29984,8 @@ static int unixWrite( } } #endif - - while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ amt -= wrote; offset += wrote; pBuf = &((char*)pBuf)[wrote]; @@ -28166,12 +29993,12 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>0 ){ + if( amt>wrote ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; }else{ - pFile->lastErrno = 0; /* not a system error */ + storeLastErrno(pFile, 0); /* not a system error */ return SQLITE_FULL; } } @@ -28192,9 +30019,9 @@ SQLITE_API int sqlite3_fullsync_count = 0; ** We do not trust systems to provide a working fdatasync(). Some do. ** Others do no. To be safe, we will stick with the (slightly slower) ** fsync(). If you know that your system does support fdatasync() correctly, -** then simply compile with -Dfdatasync=fdatasync +** then simply compile with -Dfdatasync=fdatasync or -DHAVE_FDATASYNC */ -#if !defined(fdatasync) +#if !defined(fdatasync) && !HAVE_FDATASYNC # define fdatasync fsync #endif @@ -28380,7 +30207,7 @@ static int unixSync(sqlite3_file *id, int flags){ rc = full_fsync(pFile->h, isFullsync, isDataOnly); SimulateIOError( rc=1 ); if( rc ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); } @@ -28424,7 +30251,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ rc = robust_ftruncate(pFile->h, nByte); if( rc ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); }else{ #ifdef SQLITE_DEBUG @@ -28464,7 +30291,7 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ rc = osFstat(((unixFile*)id)->h, &buf); SimulateIOError( rc=1 ); if( rc!=0 ){ - ((unixFile*)id)->lastErrno = errno; + storeLastErrno((unixFile*)id, errno); return SQLITE_IOERR_FSTAT; } *pSize = buf.st_size; @@ -28500,7 +30327,9 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ - if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + if( osFstat(pFile->h, &buf) ){ + return SQLITE_IOERR_FSTAT; + } nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; if( nSize>(i64)buf.st_size ){ @@ -28515,24 +30344,28 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ }while( err==EINTR ); if( err ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. This - ** is the same technique used by glibc to implement posix_fallocate() - ** on systems that do not have a real fallocate() system call. + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. */ int nBlk = buf.st_blksize; /* File-system block size */ + int nWrite = 0; /* Number of bytes written by seekAndWrite */ i64 iWrite; /* Next offset to write to */ - if( robust_ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); - } iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - while( iWrite=buf.st_size ); + assert( (iWrite/nBlk)==((buf.st_size+nBlk-1)/nBlk) ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWriteszChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); } } @@ -28581,11 +30414,15 @@ static int unixGetTempname(int nBuf, char *zBuf); static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ + case SQLITE_FCNTL_WAL_BLOCK: { + /* pFile->ctrlFlags |= UNIXFILE_BLOCK; // Deferred feature */ + return SQLITE_OK; + } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = pFile->lastErrno; return SQLITE_OK; } @@ -28613,7 +30450,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { - char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + char *zTFile = sqlite3_malloc64( pFile->pVfs->mxPathname ); if( zTFile ){ unixGetTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; @@ -28654,8 +30491,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - case SQLITE_SET_LOCKPROXYFILE: - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ @@ -28795,7 +30632,9 @@ static int unixDeviceCharacteristics(sqlite3_file *id){ ** Instead, it should be called via macro osGetpagesize(). */ static int unixGetpagesize(void){ -#if defined(_BSD_SOURCE) +#if OS_VXWORKS + return 1024; +#elif defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); @@ -28888,15 +30727,17 @@ struct unixShm { ** otherwise. */ static int unixShmSystemLock( - unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ + unixFile *pFile, /* Open connection to the WAL file */ int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ int ofst, /* First byte of the locking range */ int n /* Number of bytes to lock */ ){ - struct flock f; /* The posix advisory locking structure */ - int rc = SQLITE_OK; /* Result code form fcntl() */ + unixShmNode *pShmNode; /* Apply locks to this open shared-memory segment */ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ /* Access to the unixShmNode object is serialized by the caller */ + pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); /* Shared locks never span more than one byte */ @@ -28906,6 +30747,7 @@ static int unixShmSystemLock( assert( n>=1 && nh>=0 ){ + int lkType; /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); f.l_type = lockType; @@ -28913,8 +30755,10 @@ static int unixShmSystemLock( f.l_start = ofst; f.l_len = n; - rc = osFcntl(pShmNode->h, F_SETLK, &f); + lkType = (pFile->ctrlFlags & UNIXFILE_BLOCK)!=0 ? F_SETLKW : F_SETLK; + rc = osFcntl(pShmNode->h, lkType, &f); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; + pFile->ctrlFlags &= ~UNIXFILE_BLOCK; } /* Update the global lock state and do debug tracing */ @@ -29047,7 +30891,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); @@ -29060,6 +30904,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode = pInode->pShmNode; if( pShmNode==0 ){ struct stat sStat; /* fstat() info for database file */ +#ifndef SQLITE_SHM_DIRECTORY + const char *zBasePath = pDbFd->zPath; +#endif /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it @@ -29073,9 +30920,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ #ifdef SQLITE_SHM_DIRECTORY nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; #else - nShmFilename = 6 + (int)strlen(pDbFd->zPath); + nShmFilename = 6 + (int)strlen(zBasePath); #endif - pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; @@ -29087,7 +30934,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); #endif pShmNode->h = -1; @@ -29121,13 +30968,13 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** If not, truncate the file to zero length. */ rc = SQLITE_OK; - if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); } if( rc ) goto shm_open_err; } @@ -29285,7 +31132,7 @@ static int unixShmMap( goto shmpage_out; } }else{ - pMem = sqlite3_malloc(szRegion); + pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ rc = SQLITE_NOMEM; goto shmpage_out; @@ -29359,7 +31206,7 @@ static int unixShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -29387,7 +31234,7 @@ static int unixShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -29412,7 +31259,7 @@ static int unixShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); + rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -29421,7 +31268,7 @@ static int unixShmLock( } sqlite3_mutex_leave(pShmNode->mutex); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", - p->id, getpid(), p->sharedMask, p->exclMask)); + p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; } @@ -29435,7 +31282,8 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - unixEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -29480,7 +31328,9 @@ static int unixShmUnmap( assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->h>=0 ) osUnlink(pShmNode->zFilename); + if( deleteFlag && pShmNode->h>=0 ){ + osUnlink(pShmNode->zFilename); + } unixShmPurge(pDbFd); } unixLeaveMutex(); @@ -29757,7 +31607,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \ +#define IOMETHODS(FINDER,METHOD,VERSION,CLOSE,LOCK,UNLOCK,CKLOCK,SHMMAP) \ static const sqlite3_io_methods METHOD = { \ VERSION, /* iVersion */ \ CLOSE, /* xClose */ \ @@ -29822,7 +31672,7 @@ IOMETHODS( 0 /* xShmMap method */ ) -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +#if SQLITE_ENABLE_LOCKING_STYLE IOMETHODS( flockIoFinder, /* Finder function name */ flockIoMethods, /* sqlite3_io_methods object name */ @@ -29840,10 +31690,10 @@ IOMETHODS( semIoFinder, /* Finder function name */ semIoMethods, /* sqlite3_io_methods object name */ 1, /* shared memory is disabled */ - semClose, /* xClose method */ - semLock, /* xLock method */ - semUnlock, /* xUnlock method */ - semCheckReservedLock, /* xCheckReservedLock method */ + semXClose, /* xClose method */ + semXLock, /* xLock method */ + semXUnlock, /* xUnlock method */ + semXCheckReservedLock, /* xCheckReservedLock method */ 0 /* xShmMap method */ ) #endif @@ -29967,15 +31817,13 @@ static const sqlite3_io_methods #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ -#if OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE -/* -** This "finder" function attempts to determine the best locking strategy -** for the database file "filePath". It then returns the sqlite3_io_methods -** object that implements that strategy. -** -** This is for VXWorks only. +#if OS_VXWORKS +/* +** This "finder" function for VxWorks checks to see if posix advisory +** locking works. If it does, then that is what is used. If it does not +** work, then fallback to named semaphore locking. */ -static const sqlite3_io_methods *autolockIoFinderImpl( +static const sqlite3_io_methods *vxworksIoFinderImpl( const char *filePath, /* name of the database file */ unixFile *pNew /* the open file object */ ){ @@ -30001,9 +31849,9 @@ static const sqlite3_io_methods *autolockIoFinderImpl( } } static const sqlite3_io_methods - *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; -#endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ +#endif /* OS_VXWORKS */ /* ** An abstract type for a pointer to an IO method finder function: @@ -30122,7 +31970,7 @@ static int fillInUnixFile( ** the afpLockingContext. */ afpLockingContext *pCtx; - pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ rc = SQLITE_NOMEM; }else{ @@ -30152,7 +32000,7 @@ static int fillInUnixFile( int nFilename; assert( zFilename!=0 ); nFilename = (int)strlen(zFilename) + 6; - zLockFile = (char *)sqlite3_malloc(nFilename); + zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ rc = SQLITE_NOMEM; }else{ @@ -30185,7 +32033,7 @@ static int fillInUnixFile( } #endif - pNew->lastErrno = 0; + storeLastErrno(pNew, 0); #if OS_VXWORKS if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); @@ -30516,8 +32364,8 @@ static int unixOpen( ** the same instant might all reset the PRNG. But multiple resets ** are harmless. */ - if( randomnessPid!=getpid() ){ - randomnessPid = getpid(); + if( randomnessPid!=osGetpid(0) ){ + randomnessPid = osGetpid(0); sqlite3_randomness(0,0); } @@ -30529,7 +32377,7 @@ static int unixOpen( if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM; } @@ -30633,13 +32481,16 @@ static int unixOpen( #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ - ((unixFile*)pFile)->lastErrno = errno; + storeLastErrno(p, errno); robust_close(p, fd, __LINE__); return SQLITE_IOERR_ACCESS; } if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } + if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } #endif /* Set up appropriate ctrlFlags */ @@ -30662,19 +32513,6 @@ static int unixOpen( if( envforce!=NULL ){ useProxy = atoi(envforce)>0; }else{ - if( statfs(zPath, &fsInfo) == -1 ){ - /* In theory, the close(fd) call is sub-optimal. If the file opened - ** with fd is a database file, and there are other connections open - ** on that file that are currently holding advisory locks on it, - ** then the call to close() will cancel those locks. In practice, - ** we're assuming that statfs() doesn't fail very often. At least - ** not while other file descriptors opened by the same process on - ** the same file are working. */ - p->lastErrno = errno; - robust_close(p, fd, __LINE__); - rc = SQLITE_IOERR_ACCESS; - goto open_finished; - } useProxy = !(fsInfo.f_flags&MNT_LOCAL); } if( useProxy ){ @@ -30918,8 +32756,8 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** tests repeatable. */ memset(zBuf, 0, nBuf); - randomnessPid = getpid(); -#if !defined(SQLITE_TEST) + randomnessPid = osGetpid(0); +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) { int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); @@ -31100,9 +32938,10 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** ** C APIs ** -** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_SET_LOCKPROXYFILE, ** | ":auto:"); -** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_GET_LOCKPROXYFILE, +** &); ** ** ** SQL pragmas @@ -31195,7 +33034,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 ** will force automatic proxy locking to be disabled for all database -** files (explicitly calling the SQLITE_SET_LOCKPROXYFILE pragma or +** files (explicitly calling the SQLITE_FCNTL_SET_LOCKPROXYFILE pragma or ** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). */ @@ -31216,6 +33055,7 @@ struct proxyLockingContext { char *lockProxyPath; /* Name of the proxy lock file */ char *dbPath; /* Name of the open file */ int conchHeld; /* 1 if the conch is held, -1 if lockless */ + int nFails; /* Number of conch taking failures */ void *oldLockingContext; /* Original lockingcontext to restore on close */ sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ }; @@ -31237,7 +33077,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ { if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", - lPath, errno, getpid())); + lPath, errno, osGetpid(0))); return SQLITE_IOERR_LOCK; } len = strlcat(lPath, "sqliteplocks", maxLen); @@ -31259,7 +33099,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ } lPath[i+len]='\0'; strlcat(lPath, ":auto:", maxLen); - OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lPath, getpid())); + OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lPath, osGetpid(0))); return SQLITE_OK; } @@ -31286,7 +33126,7 @@ static int proxyCreateLockPath(const char *lockPath){ if( err!=EEXIST ) { OSTRACE(("CREATELOCKPATH FAILED creating %s, " "'%s' proxy lock path=%s pid=%d\n", - buf, strerror(err), lockPath, getpid())); + buf, strerror(err), lockPath, osGetpid(0))); return err; } } @@ -31295,7 +33135,7 @@ static int proxyCreateLockPath(const char *lockPath){ } buf[i] = lockPath[i]; } - OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, osGetpid(0))); return 0; } @@ -31329,7 +33169,7 @@ static int proxyCreateUnixFile( if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM; } @@ -31362,7 +33202,7 @@ static int proxyCreateUnixFile( } } - pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ rc = SQLITE_NOMEM; goto end_create_proxy; @@ -31395,8 +33235,10 @@ SQLITE_API int sqlite3_hostid_num = 0; #define PROXY_HOSTIDLEN 16 /* conch file host id length */ +#ifdef HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); +#endif /* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN ** bytes of writable memory. @@ -31404,10 +33246,9 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait); static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); -#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ - && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 +#ifdef HAVE_GETHOSTUUID { - static const struct timespec timeout = {1, 0}; /* 1 sec timeout */ + struct timespec timeout = {1, 0}; /* 1 sec timeout */ if( gethostuuid(pHostID, &timeout) ){ int err = errno; if( pError ){ @@ -31522,7 +33363,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ */ struct stat buf; if( osFstat(conchFile->h, &buf) ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } @@ -31542,7 +33383,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ char tBuf[PROXY_MAXCONCHLEN]; int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ - pFile->lastErrno = errno; + storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ @@ -31562,7 +33403,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ if( 0==proxyBreakConchLock(pFile, myHostID) ){ rc = SQLITE_OK; if( lockType==EXCLUSIVE_LOCK ){ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); } if( !rc ){ rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); @@ -31600,11 +33441,12 @@ static int proxyTakeConch(unixFile *pFile){ int forceNewLockPath = 0; OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid())); + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); rc = proxyGetHostID(myHostID, &pError); if( (rc&0xff)==SQLITE_IOERR ){ - pFile->lastErrno = pError; + storeLastErrno(pFile, pError); goto end_takeconch; } rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); @@ -31615,7 +33457,7 @@ static int proxyTakeConch(unixFile *pFile){ readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); if( readLen<0 ){ /* I/O error: lastErrno set by seekAndRead */ - pFile->lastErrno = conchFile->lastErrno; + storeLastErrno(pFile, conchFile->lastErrno); rc = SQLITE_IOERR_READ; goto end_takeconch; }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || @@ -31688,7 +33530,7 @@ static int proxyTakeConch(unixFile *pFile){ rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } }else{ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } if( rc==SQLITE_OK ){ char writeBuffer[PROXY_MAXCONCHLEN]; @@ -31697,7 +33539,8 @@ static int proxyTakeConch(unixFile *pFile){ writeBuffer[0] = (char)PROXY_CONCHVERSION; memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); if( pCtx->lockProxyPath!=NULL ){ - strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, + MAXPATHLEN); }else{ strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); } @@ -31809,7 +33652,7 @@ static int proxyReleaseConch(unixFile *pFile){ conchFile = pCtx->conchFile; OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), - getpid())); + osGetpid(0))); if( pCtx->conchHeld>0 ){ rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); } @@ -31821,7 +33664,7 @@ static int proxyReleaseConch(unixFile *pFile){ /* ** Given the name of a database file, compute the name of its conch file. -** Store the conch filename in memory obtained from sqlite3_malloc(). +** Store the conch filename in memory obtained from sqlite3_malloc64(). ** Make *pConchPath point to the new name. Return SQLITE_OK on success ** or SQLITE_NOMEM if unable to obtain memory. ** @@ -31837,7 +33680,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ /* Allocate space for the conch filename and initialize the name to ** the name of the original database file. */ - *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ return SQLITE_NOMEM; } @@ -31909,7 +33752,8 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ /* afp style keeps a reference to the db path in the filePath field ** of the struct */ assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, + MAXPATHLEN); } else #endif if( pFile->pMethod == &dotlockIoMethods ){ @@ -31950,9 +33794,9 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { } OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, - (lockPath ? lockPath : ":auto:"), getpid())); + (lockPath ? lockPath : ":auto:"), osGetpid(0))); - pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ return SQLITE_NOMEM; } @@ -32022,7 +33866,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { */ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ switch( op ){ - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; if( pFile->pMethod == &proxyIoMethods ){ proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; @@ -32037,13 +33881,16 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_SET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; int rc = SQLITE_OK; int isProxyStyle = (pFile->pMethod == &proxyIoMethods); if( pArg==NULL || (const char *)pArg==0 ){ if( isProxyStyle ){ - /* turn off proxy locking - not supported */ + /* turn off proxy locking - not supported. If support is added for + ** switching proxy locking mode off then it will need to fail if + ** the journal mode is WAL mode. + */ rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; }else{ /* turn off proxy locking - already off - NOOP */ @@ -32234,7 +34081,7 @@ static int proxyClose(sqlite3_file *id) { ** necessarily been initialized when this routine is called, and so they ** should not be used. */ -SQLITE_API int sqlite3_os_init(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ /* ** The following macro defines an initializer for an sqlite3_vfs object. ** The name of the VFS is NAME. The pAppData is a pointer to a pointer @@ -32288,8 +34135,10 @@ SQLITE_API int sqlite3_os_init(void){ ** array cannot be const. */ static sqlite3_vfs aVfs[] = { -#if SQLITE_ENABLE_LOCKING_STYLE && (OS_VXWORKS || defined(__APPLE__)) +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) UNIXVFS("unix", autolockIoFinder ), +#elif OS_VXWORKS + UNIXVFS("unix", vxworksIoFinder ), #else UNIXVFS("unix", posixIoFinder ), #endif @@ -32299,11 +34148,11 @@ SQLITE_API int sqlite3_os_init(void){ #if OS_VXWORKS UNIXVFS("unix-namedsem", semIoFinder ), #endif -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS UNIXVFS("unix-posix", posixIoFinder ), -#if !OS_VXWORKS - UNIXVFS("unix-flock", flockIoFinder ), #endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-flock", flockIoFinder ), #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) UNIXVFS("unix-afp", afpIoFinder ), @@ -32331,7 +34180,7 @@ SQLITE_API int sqlite3_os_init(void){ ** to release dynamically allocated objects. But not on unix. ** This routine is a no-op for unix. */ -SQLITE_API int sqlite3_os_end(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ return SQLITE_OK; } @@ -32353,6 +34202,7 @@ SQLITE_API int sqlite3_os_end(void){ ** ** This file contains code that is specific to Windows. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* This file is used for Windows only */ /* @@ -32391,16 +34241,6 @@ SQLITE_API int sqlite3_os_end(void){ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. @@ -32571,6 +34411,7 @@ SQLITE_API int sqlite3_open_file_count = 0; /* ** Include the header file for the Windows VFS. */ +/* #include "os_win.h" */ /* ** Compiling and using WAL mode requires several APIs that are only @@ -32581,6 +34422,11 @@ SQLITE_API int sqlite3_open_file_count = 0; with SQLITE_OMIT_WAL." #endif +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + /* ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? @@ -32710,10 +34556,11 @@ SQLITE_API int sqlite3_open_file_count = 0; /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL -** mode (e.g. these APIs are available in the Windows CE SDK; however, they -** are not present in the header file)? +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? */ -#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) /* ** Two of the file mapping APIs are different under WinRT. Figure out which ** set we need. @@ -32738,10 +34585,12 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); #endif /* SQLITE_OS_WINRT */ /* -** This file mapping API is common to both Win32 and WinRT. +** These file mapping APIs are common to both Win32 and WinRT. */ + +WINBASEAPI BOOL WINAPI FlushViewOfFile(LPCVOID, SIZE_T); WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); -#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ +#endif /* SQLITE_WIN32_FILEMAPPING_API */ /* ** Some Microsoft compilers lack this definition. @@ -33034,7 +34883,7 @@ static struct win_syscall { LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) #if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -33044,7 +34893,7 @@ static struct win_syscall { DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -33384,7 +35233,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -33454,7 +35304,7 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[58].pCurrent) -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, #else { "UnmapViewOfFile", (SYSCALL)0, 0 }, @@ -33517,7 +35367,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -33581,7 +35431,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, @@ -33606,6 +35456,32 @@ static struct win_syscall { SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) #endif /* defined(InterlockedCompareExchange) */ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreate", (SYSCALL)UuidCreate, 0 }, +#else + { "UuidCreate", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, +#else + { "UuidCreateSequential", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreateSequential \ + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + +#if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 + { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, +#else + { "FlushViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osFlushViewOfFile \ + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -33699,7 +35575,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** "pnLargest" argument, if non-zero, will be used to return the size of the ** largest committed free block in the heap, in bytes. */ -SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_compact_heap(LPUINT pnLargest){ int rc = SQLITE_OK; UINT nLargest = 0; HANDLE hHeap; @@ -33739,12 +35615,12 @@ SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ ** the sqlite3_memory_used() function does not return zero, SQLITE_BUSY will ** be returned and no changes will be made to the Win32 native heap. */ -SQLITE_API int sqlite3_win32_reset_heap(){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -33784,7 +35660,7 @@ SQLITE_API int sqlite3_win32_reset_heap(){ ** (if available). */ -SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ @@ -33824,7 +35700,7 @@ SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ static HANDLE sleepObj = NULL; #endif -SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds){ #if SQLITE_OS_WINRT if ( sleepObj==NULL ){ sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, @@ -33873,7 +35749,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** This function determines if the machine is running a version of Windows ** based on the NT kernel. */ -SQLITE_API int sqlite3_win32_is_nt(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void){ #if SQLITE_OS_WINRT /* ** NOTE: The WinRT sub-platform is always assumed to be based on the NT @@ -34227,7 +36103,7 @@ static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ ** Convert multibyte character string to UTF-8. Space to hold the ** returned string is obtained from sqlite3_malloc(). */ -SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ +SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8(const char *zFilename){ char *zFilenameUtf8; LPWSTR zTmpWide; @@ -34244,7 +36120,7 @@ SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ ** Convert UTF-8 to multibyte character string. Space to hold the ** returned string is obtained from sqlite3_malloc(). */ -SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ +SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs(const char *zFilename){ char *zFilenameMbcs; LPWSTR zTmpWide; @@ -34264,7 +36140,7 @@ SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ ** argument is the name of the directory to use. The return value will be ** SQLITE_OK if successful. */ -SQLITE_API int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ char **ppDirectory = 0; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); @@ -34489,11 +36365,11 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ /* ** Log a I/O error retry episode. */ -static void winLogIoerr(int nRetry){ +static void winLogIoerr(int nRetry, int lineno){ if( nRetry ){ - sqlite3_log(SQLITE_IOERR, - "delayed %dms for lock/sharing conflict", - winIoerrRetryDelay*nRetry*(nRetry+1)/2 + sqlite3_log(SQLITE_NOTICE, + "delayed %dms for lock/sharing conflict at line %d", + winIoerrRetryDelay*nRetry*(nRetry+1)/2, lineno ); } } @@ -34973,7 +36849,8 @@ static int winClose(sqlite3_file *id){ assert( pFile->pShm==0 ); #endif assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); - OSTRACE(("CLOSE file=%p\n", pFile->h)); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p\n", + osGetCurrentProcessId(), pFile, pFile->h)); #if SQLITE_MAX_MMAP_SIZE>0 winUnmapfile(pFile); @@ -35002,7 +36879,8 @@ static int winClose(sqlite3_file *id){ pFile->h = NULL; } OpenCounter(-1); - OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -35019,7 +36897,7 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for ReadFile. */ #endif winFile *pFile = (winFile*)id; /* file handle */ @@ -35030,7 +36908,8 @@ static int winRead( assert( amt>0 ); assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 @@ -35039,7 +36918,8 @@ static int winRead( if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); - OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("READ-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -35051,9 +36931,10 @@ static int winRead( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( winSeekFile(pFile, offset) ){ - OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -35067,19 +36948,22 @@ static int winRead( DWORD lastErrno; if( winRetryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; - OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } - winLogIoerr(nRetry); + winLogIoerr(nRetry, __LINE__); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); - OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_SHORT_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_IOERR_SHORT_READ; } - OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; } @@ -35102,7 +36986,8 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 @@ -35111,7 +36996,8 @@ static int winWrite( if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); - OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("WRITE-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -35123,13 +37009,13 @@ static int winWrite( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) rc = winSeekFile(pFile, offset); if( rc==0 ){ #else { #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for WriteFile. */ #endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ @@ -35137,14 +37023,14 @@ static int winWrite( DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); #endif while( nRem>0 ){ -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ #else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ @@ -35157,7 +37043,7 @@ static int winWrite( lastErrno = osGetLastError(); break; } -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) offset += nWrite; overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); @@ -35174,17 +37060,20 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ - OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_FULL, pFile->lastErrno, "winWrite1", pFile->zPath); } - OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_WRITE\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite2", pFile->zPath); }else{ - winLogIoerr(nRetry); + winLogIoerr(nRetry, __LINE__); } - OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; } @@ -35198,8 +37087,8 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", - pFile->h, nByte, pFile->locktype)); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, size=%lld, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -35231,7 +37120,8 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #endif - OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, sqlite3ErrName(rc))); return rc; } @@ -35255,7 +37145,7 @@ static int winSync(sqlite3_file *id, int flags){ BOOL rc; #endif #if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ - (defined(SQLITE_TEST) && defined(SQLITE_DEBUG)) + defined(SQLITE_HAVE_OS_TRACE) /* ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or ** OSTRACE() macros. @@ -35276,8 +37166,9 @@ static int winSync(sqlite3_file *id, int flags){ */ SimulateDiskfullError( return SQLITE_FULL ); - OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", - pFile->h, flags, pFile->locktype)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, flags=%x, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, flags, + pFile->locktype)); #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); @@ -35292,19 +37183,38 @@ static int winSync(sqlite3_file *id, int flags){ ** no-op */ #ifdef SQLITE_NO_SYNC - OSTRACE(("SYNC-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("SYNC-NOP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; #else +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + if( osFlushViewOfFile(pFile->pMapRegion, 0) ){ + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_OK\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winSync1", pFile->zPath); + } + } +#endif rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ - OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); - OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_FSYNC\n", + osGetCurrentProcessId(), pFile, pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, - "winSync", pFile->zPath); + "winSync2", pFile->zPath); } #endif } @@ -35493,6 +37403,12 @@ static int winLock(sqlite3_file *id, int locktype){ return SQLITE_OK; } + /* Do not allow any kind of write-lock on a read-only database + */ + if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ + return SQLITE_IOERR_LOCK; + } + /* Make sure the locking sequence is correct */ assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); @@ -35862,14 +37778,14 @@ static SYSTEM_INFO winSysInfo; ** winShmLeaveMutex() */ static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif @@ -35912,7 +37828,7 @@ struct winShmNode { int nRef; /* Number of winShm objects pointing to this */ winShm *pFirst; /* All winShm objects pointing to this */ winShmNode *pNext; /* Next in list of all winShmNode objects */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ #endif }; @@ -35943,7 +37859,7 @@ struct winShm { u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif }; @@ -36134,7 +38050,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Make the new connection a child of the winShmNode */ p->pShmNode = pShmNode; -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) p->id = pShmNode->nextShmId++; #endif pShmNode->nRef++; @@ -36322,8 +38238,8 @@ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ winShmLeaveMutex(); } @@ -36354,16 +38270,16 @@ static int winShmMap( void volatile **pp /* OUT: Mapped memory */ ){ winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; + winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; int rc = SQLITE_OK; - if( !p ){ + if( !pShm ){ rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; - p = pDbFd->pShm; + pShm = pDbFd->pShm; } - pShmNode = p->pShmNode; + pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); @@ -36403,7 +38319,7 @@ static int winShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc( + apNew = (struct ShmRegion *)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ @@ -37275,7 +39191,7 @@ static int winOpen( } } #endif - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); @@ -37459,7 +39375,7 @@ static int winDelete( if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); } sqlite3_free(zConverted); OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); @@ -37509,7 +39425,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ - winLogIoerr(cnt); + winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); return winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", @@ -37850,7 +39766,7 @@ static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ int n = 0; UNUSED_PARAMETER(pVfs); -#if defined(SQLITE_TEST) +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) n = nBuf; memset(zBuf, 0, nBuf); #else @@ -37884,7 +39800,23 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ memcpy(&zBuf[n], &i, sizeof(i)); n += sizeof(i); } +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + if( sizeof(UUID)<=nBuf-n ){ + UUID id; + memset(&id, 0, sizeof(UUID)); + osUuidCreate(&id); + memcpy(&zBuf[n], &id, sizeof(UUID)); + n += sizeof(UUID); + } + if( sizeof(UUID)<=nBuf-n ){ + UUID id; + memset(&id, 0, sizeof(UUID)); + osUuidCreateSequential(&id); + memcpy(&zBuf[n], &id, sizeof(UUID)); + n += sizeof(UUID); + } #endif +#endif /* defined(SQLITE_TEST) || defined(SQLITE_ZERO_PRNG_SEED) */ return n; } @@ -38008,7 +39940,7 @@ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ /* ** Initialize and deinitialize the operating system interface. */ -SQLITE_API int sqlite3_os_init(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ static sqlite3_vfs winVfs = { 3, /* iVersion */ sizeof(winFile), /* szOsFile */ @@ -38062,7 +39994,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==77 ); + assert( ArraySize(aSyscall)==80 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); @@ -38083,7 +40015,7 @@ SQLITE_API int sqlite3_os_init(void){ return SQLITE_OK; } -SQLITE_API int sqlite3_os_end(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ #if SQLITE_OS_WINRT if( sleepObj!=NULL ){ osCloseHandle(sleepObj); @@ -38133,6 +40065,7 @@ SQLITE_API int sqlite3_os_end(void){ ** start of a transaction, and is thus usually less than a few thousand, ** but can be as large as 2 billion for a really big database. */ +/* #include "sqliteInt.h" */ /* Size of the Bitvec structure in bytes. */ #define BITVEC_SZ 512 @@ -38224,10 +40157,10 @@ SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ ** If p is NULL (if the bitmap has not been created) or if ** i is out of range, then return false. */ -SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ - if( p==0 ) return 0; - if( i>p->iSize || i==0 ) return 0; +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ + assert( p!=0 ); i--; + if( i>=p->iSize ) return 0; while( p->iDivisor ){ u32 bin = i/p->iDivisor; i = i%p->iDivisor; @@ -38247,6 +40180,9 @@ SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ return 0; } } +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + return p!=0 && sqlite3BitvecTestNotNull(p,i); +} /* ** Set the i-th bit. Return 0 on success and an error code if @@ -38439,7 +40375,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ** bits to act as the reference */ pBitvec = sqlite3BitvecCreate( sz ); pV = sqlite3MallocZero( (sz+7)/8 + 1 ); - pTmpSpace = sqlite3_malloc(BITVEC_SZ); + pTmpSpace = sqlite3_malloc64(BITVEC_SZ); if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; /* NULL pBitvec tests */ @@ -38519,6 +40455,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ************************************************************************* ** This file implements that page cache. */ +/* #include "sqliteInt.h" */ /* ** A complete page cache is an instance of this structure. @@ -38526,7 +40463,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ @@ -38535,21 +40472,8 @@ struct PCache { int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ void *pStress; /* Argument to xStress */ sqlite3_pcache *pCache; /* Pluggable cache module */ - PgHdr *pPage1; /* Reference to page 1 */ }; -/* -** Some of the assert() macros in this code are too expensive to run -** even during normal debugging. Use them only rarely on long-running -** tests. Enable the expensive asserts using the -** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. -*/ -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT -# define expensive_assert(X) assert(X) -#else -# define expensive_assert(X) -#endif - /********************************** Linked List Management ********************/ /* Allowed values for second argument to pcacheManageDirtyList() */ @@ -38625,20 +40549,25 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ */ static void pcacheUnpin(PgHdr *p){ if( p->pCache->bPurgeable ){ - if( p->pgno==1 ){ - p->pCache->pPage1 = 0; - } sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); } } /* -** Compute the number of pages of cache requested. +** Compute the number of pages of cache requested. p->szCache is the +** cache size requested by the "PRAGMA cache_size" statement. +** +** */ static int numberOfCachePages(PCache *p){ if( p->szCache>=0 ){ + /* IMPLEMENTATION-OF: R-42059-47211 If the argument N is positive then the + ** suggested cache size is set to N. */ return p->szCache; }else{ + /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then + ** the number of cache pages is adjusted to use approximately abs(N*1024) + ** bytes of memory. */ return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } } @@ -38699,11 +40628,12 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( - szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable ); if( pNew==0 ) return SQLITE_NOMEM; sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); @@ -38711,7 +40641,6 @@ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } pCache->pCache = pNew; - pCache->pPage1 = 0; pCache->szPage = szPage; } return SQLITE_OK; @@ -38836,13 +40765,14 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(pPgHdr, 0, sizeof(PgHdr)); + memset(pPgHdr, 0, sizeof(PgHdr)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, pCache->szExtra); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; + pPgHdr->flags = PGHDR_CLEAN; return sqlite3PcacheFetchFinish(pCache,pgno,pPage); } @@ -38859,19 +40789,14 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( ){ PgHdr *pPgHdr; - if( pPage==0 ) return 0; + assert( pPage!=0 ); pPgHdr = (PgHdr *)pPage->pExtra; if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } + pCache->nRefSum++; pPgHdr->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPgHdr; - } return pPgHdr; } @@ -38881,10 +40806,9 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( */ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - p->pCache->nRef--; - if( (p->flags&PGHDR_DIRTY)==0 ){ + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ + if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); }else if( p->pDirtyPrev!=0 ){ /* Move the page to the head of the dirty list. */ @@ -38899,6 +40823,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); p->nRef++; + p->pCache->nRefSum++; } /* @@ -38911,10 +40836,7 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRef--; - if( p->pgno==1 ){ - p->pCache->pPage1 = 0; - } + p->pCache->nRefSum--; sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -38923,11 +40845,14 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ - p->flags &= ~PGHDR_DONT_WRITE; assert( p->nRef>0 ); - if( 0==(p->flags & PGHDR_DIRTY) ){ - p->flags |= PGHDR_DIRTY; - pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ + p->flags &= ~PGHDR_DONT_WRITE; + if( p->flags & PGHDR_CLEAN ){ + p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + } } } @@ -38937,8 +40862,10 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ if( (p->flags & PGHDR_DIRTY) ){ + assert( (p->flags & PGHDR_CLEAN)==0 ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; if( p->nRef==0 ){ pcacheUnpin(p); } @@ -39005,9 +40932,14 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->pPage1 ){ - memset(pCache->pPage1->pData, 0, pCache->szPage); - pgno = 1; + if( pgno==0 && pCache->nRefSum ){ + sqlite3_pcache_page *pPage1; + pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); + if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because + ** pCache->nRefSum>0 */ + memset(pPage1->pBuf, 0, pCache->szPage); + pgno = 1; + } } sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); } @@ -39110,10 +41042,13 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ } /* -** Return the total number of referenced pages held by the cache. +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. */ SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; + return pCache->nRefSum; } /* @@ -39158,6 +41093,13 @@ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } + + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified @@ -39191,14 +41133,96 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overridden, then neither of ** these two features are available. +** +** A Page cache line looks like this: +** +** ------------------------------------------------------------- +** | database page content | PgHdr1 | MemPage | PgHdr | +** ------------------------------------------------------------- +** +** The database page content is up front (so that buffer overreads tend to +** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage +** is the extension added by the btree.c module containing information such +** as the database page number and how that database page is used. PgHdr +** is added by the pcache.c layer and contains information used to keep track +** of which pages are "dirty". PgHdr1 is an extension added by this +** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. +** PgHdr1 contains information needed to look up a page by its page number. +** The superclass sqlite3_pcache_page.pBuf points to the start of the +** database page content and sqlite3_pcache_page.pExtra points to PgHdr. +** +** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at +** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The +** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this +** size can vary according to architecture, compile-time options, and +** SQLite library version number. +** +** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained +** using a separate memory allocation from the database page content. This +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more +** than a power of two with the memory allocator rounding up to the next +** power of two, and leaving the rounded-up space unused. +** +** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates +** with this module. Information is passed back and forth as PgHdr1 pointers. +** +** The pcache.c and pager.c modules deal pointers to PgHdr objects. +** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, 0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. */ - +/* #include "sqliteInt.h" */ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -39227,7 +41251,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -39245,8 +41269,9 @@ struct PCache1 { ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ - int szPage; /* Size of allocated pages in bytes */ - int szExtra; /* Size of extra space in bytes */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ @@ -39260,27 +41285,13 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ }; /* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - -/* -** Free slots in the allocator used to divide up the buffer provided using -** the SQLITE_CONFIG_PAGECACHE mechanism. +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. */ struct PgFreeslot { PgFreeslot *pNext; /* Next free slot */ @@ -39298,10 +41309,12 @@ static SQLITE_WSD struct PCacheGlobal { ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ @@ -39323,12 +41336,20 @@ static SQLITE_WSD struct PCacheGlobal { /* ** Macros to enter and leave the PCache LRU mutex. */ -#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) -#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 +# define pcache1EnterMutex(X) assert((X)->mutex==0) +# define pcache1LeaveMutex(X) assert((X)->mutex==0) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 +#else +# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 +#endif /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -39341,6 +41362,7 @@ static SQLITE_WSD struct PCacheGlobal { SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; @@ -39358,6 +41380,44 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ } } +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + i64 szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; + }else{ + szBulk = -1024 * (i64)pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ + szBulk = pCache->szAlloc*pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + int i; + for(i=0; iszPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->isAnchor = 0; + pX->pNext = pCache->pFree; + pCache->pFree = pX; + zBulk += pCache->szAlloc; + } + } + return pCache->pFree!=0; +} + /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no @@ -39370,7 +41430,6 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ static void *pcache1Alloc(int nByte){ void *p = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); if( nByte<=pcache1.szSlot ){ sqlite3_mutex_enter(pcache1.mutex); p = (PgHdr1 *)pcache1.pFree; @@ -39379,7 +41438,8 @@ static void *pcache1Alloc(int nByte){ pcache1.nFreeSlot--; pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); } sqlite3_mutex_leave(pcache1.mutex); } @@ -39392,7 +41452,8 @@ static void *pcache1Alloc(int nByte){ if( p ){ int sz = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); sqlite3_mutex_leave(pcache1.mutex); } #endif @@ -39404,13 +41465,13 @@ static void *pcache1Alloc(int nByte){ /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static int pcache1Free(void *p){ +static void pcache1Free(void *p){ int nFreed = 0; - if( p==0 ) return 0; + if( p==0 ) return; if( p>=pcache1.pStart && ppNext = pcache1.pFree; pcache1.pFree = pSlot; @@ -39421,15 +41482,14 @@ static int pcache1Free(void *p){ }else{ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - nFreed = sqlite3MallocSize(p); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + nFreed = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -nFreed); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); sqlite3_mutex_leave(pcache1.mutex); #endif sqlite3_free(p); } - return nFreed; } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT @@ -39453,58 +41513,72 @@ static int pcache1MemSize(void *p){ /* ** Allocate a new page object initially associated with cache pCache. */ -static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ +static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ PgHdr1 *p = 0; void *pPg; - /* The group mutex must be released before pcache1Alloc() is called. This - ** is because it may call sqlite3_release_memory(), which assumes that - ** this mutex is not held. */ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - pcache1LeaveMutex(pCache->pGroup); + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); +#endif + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } #ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } #else - pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; + pPg = pcache1Alloc(pCache->szAlloc); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif - pcache1EnterMutex(pCache->pGroup); - - if( pPg ){ + if( benignMalloc ){ sqlite3EndBenignMalloc(); } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); +#endif + if( pPg==0 ) return 0; p->page.pBuf = pPg; p->page.pExtra = &p[1]; - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } - return p; + p->isBulkLocal = 0; + p->isAnchor = 0; } - return 0; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } + return p; } /* ** Free a page object allocated by pcache1AllocPage(). -** -** The pointer is allowed to be NULL, which is prudent. But it turns out -** that the current implementation happens to never call this routine -** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ static void pcache1FreePage(PgHdr1 *p){ - if( ALWAYS(p) ){ - PCache1 *pCache = p->pCache; - assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; - } + } + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; } } @@ -39599,41 +41673,35 @@ static void pcache1ResizeHash(PCache1 *p){ ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1PinPage(PgHdr1 *pPage){ +static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ PCache1 *pCache; - PGroup *pGroup; assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - pGroup = pCache->pGroup; - assert( pPage->pLruNext || pPage==pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pGroup->pLruHead ); - assert( sqlite3_mutex_held(pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pGroup->pLruTail = pPage->pLruPrev; - } + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; + return pPage; } /* ** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. +** Also free the page if freePage is true. ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1RemoveFromHash(PgHdr1 *pPage){ +static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ unsigned int h; PCache1 *pCache = pPage->pCache; PgHdr1 **pp; @@ -39644,21 +41712,28 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage){ *pp = (*pp)->pNext; pCache->nPage--; + if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ -static void pcache1EnforceMaxPage(PGroup *pGroup){ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); + } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; } } @@ -39704,10 +41779,45 @@ static int pcache1Init(void *NotUsed){ UNUSED_PARAMETER(NotUsed); assert( pcache1.isInit==0 ); memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + +#if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } +#endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; + } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; return SQLITE_OK; @@ -39737,39 +41847,26 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ - /* - ** The separateCache variable is true if each PCache has its own private - ** PGroup. In other words, separateCache is true for mode (1) where no - ** mutexing is required. - ** - ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT - ** - ** * Always use a unified cache in single-threaded applications - ** - ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) - ** use separate caches (mode-1) - */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 - const int separateCache = 0; -#else - int separateCache = sqlite3GlobalConfig.bCoreMutex>0; -#endif - assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); - sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - if( separateCache ){ + if( pcache1.separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); @@ -39801,7 +41898,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); } } @@ -39819,7 +41916,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } @@ -39872,26 +41969,17 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( assert( pCache->nHash>0 && pCache->apHash ); /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ + if( pCache->bPurgeable + && !pGroup->lru.pLruPrev->isAnchor + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ PCache1 *pOther; - pPage = pGroup->pLruTail; + pPage = pGroup->lru.pLruPrev; assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage); + pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + if( pOther->szAlloc != pCache->szAlloc ){ pcache1FreePage(pPage); pPage = 0; }else{ @@ -39903,9 +41991,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** attempt to allocate a new one. */ if( !pPage ){ - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ) sqlite3EndBenignMalloc(); + pPage = pcache1AllocPage(pCache, createFlag==1); } if( pPage ){ @@ -39979,8 +42065,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. +** +** There are two versions of this routine. pcache1FetchWithMutex() is +** the general case. pcache1FetchNoMutex() is a faster implementation for +** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper +** invokes the appropriate routine. */ -static sqlite3_pcache_page *pcache1Fetch( +static PgHdr1 *pcache1FetchNoMutex( sqlite3_pcache *p, unsigned int iKey, int createFlag @@ -39988,28 +42079,66 @@ static sqlite3_pcache_page *pcache1Fetch( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = 0; - assert( offsetof(PgHdr1,page)==0 ); - assert( pCache->bPurgeable || createFlag!=1 ); - assert( pCache->bPurgeable || pCache->nMin==0 ); - assert( pCache->bPurgeable==0 || pCache->nMin==10 ); - assert( pCache->nMin==0 || pCache->bPurgeable ); - assert( pCache->nHash>0 ); - pcache1EnterMutex(pCache->pGroup); - /* Step 1: Search the hash table for an existing entry. */ pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: Abort if no existing page is found and createFlag is 0 */ + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ if( pPage ){ - if( !pPage->isPinned ) pcache1PinPage(pPage); + if( !pPage->isPinned ){ + return pcache1PinPage(pPage); + }else{ + return pPage; + } }else if( createFlag ){ /* Steps 3, 4, and 5 implemented by this subroutine */ - pPage = pcache1FetchStage2(pCache, iKey, createFlag); + return pcache1FetchStage2(pCache, iKey, createFlag); + }else{ + return 0; } +} +#if PCACHE1_MIGHT_USE_GROUP_MUTEX +static PgHdr1 *pcache1FetchWithMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage; + + pcache1EnterMutex(pCache->pGroup); + pPage = pcache1FetchNoMutex(p, iKey, createFlag); assert( pPage==0 || pCache->iMaxKey>=iKey ); pcache1LeaveMutex(pCache->pGroup); - return (sqlite3_pcache_page*)pPage; + return pPage; +} +#endif +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ +#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) + PCache1 *pCache = (PCache1 *)p; +#endif + + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + assert( pCache->nHash>0 ); +#if PCACHE1_MIGHT_USE_GROUP_MUTEX + if( pCache->pGroup->mutex ){ + return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + }else +#endif + { + return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); + } } @@ -40034,22 +42163,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ - pcache1RemoveFromHash(pPage); - pcache1FreePage(pPage); + pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -40126,8 +42249,9 @@ static void pcache1Destroy(sqlite3_pcache *p){ assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } @@ -40156,6 +42280,19 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } + +/* +** Return the global mutex used by this PCACHE implementation. The +** sqlite3_status() routine needs access to this mutex. +*/ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void){ + return pcache1.mutex; +} + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory @@ -40170,18 +42307,20 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( pcache1.pStart==0 ){ + if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); #endif assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); } pcache1LeaveMutex(&pcache1.grp); } @@ -40202,7 +42341,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } @@ -40277,6 +42416,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** There is an added cost of O(N) when switching between TEST and ** SMALLEST primitives. */ +/* #include "sqliteInt.h" */ /* @@ -40746,6 +42886,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** another is writing. */ #ifndef SQLITE_OMIT_DISKIO +/* #include "sqliteInt.h" */ /************** Include wal.h in the middle of pager.c ***********************/ /************** Begin file wal.h *********************************************/ /* @@ -40767,6 +42908,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 #ifndef _WAL_H_ #define _WAL_H_ +/* #include "sqliteInt.h" */ /* Additional values that can be added to the sync_flags argument of ** sqlite3WalFrames(): @@ -41322,9 +43464,9 @@ struct PagerSavepoint { /* ** Bits of the Pager.doNotSpill flag. See further description below. */ -#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ -#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ -#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* ** An open page cache is an instance of struct Pager. A description of @@ -41406,11 +43548,11 @@ struct PagerSavepoint { ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. ** -** If the SPILLFLAG_NOSYNC bit is set, writing to the database from pagerStress() -** is permitted, but syncing the journal file is not. This flag is set -** by sqlite3PagerWrite() when the file-system sector-size is larger than -** the database page-size in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from +** pagerStress() is permitted, but syncing the journal file is not. +** This flag is set by sqlite3PagerWrite() when the file-system sector-size +** is larger than the database page-size in order to prevent a journal sync +** from happening in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -41512,6 +43654,8 @@ struct Pager { u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -41529,9 +43673,9 @@ struct Pager { sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 iDataVersion; /* Changes whenever database content changes */ char dbFileVers[16]; /* Changes whenever database file changes */ - u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ @@ -41672,7 +43816,7 @@ static const unsigned char aJournalMagic[] = { ** ** if( pPager->jfd->pMethods ){ ... */ -#define isOpen(pFd) ((pFd)->pMethods) +#define isOpen(pFd) ((pFd)->pMethods!=0) /* ** Return true if this pager uses a write-ahead log instead of the usual @@ -41895,19 +44039,21 @@ static int subjRequiresPage(PgHdr *pPg){ int i; for(i=0; inSavepoint; i++){ p = &pPager->aSavepoint[i]; - if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ return 1; } } return 0; } +#ifdef SQLITE_DEBUG /* ** Return true if the page is already in the journal file. */ static int pageInJournal(Pager *pPager, PgHdr *pPg){ return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); } +#endif /* ** Read a 32-bit integer from the given file descriptor. Store the integer @@ -42519,7 +44665,8 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, + iHdrOff+4+nMaster+8))) ){ return rc; } @@ -42547,10 +44694,19 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ ** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ + pPager->iDataVersion++; sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } +/* +** Return the pPager->iDataVersion value +*/ +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ + assert( pPager->eState>PAGER_OPEN ); + return pPager->iDataVersion; +} + /* ** Free all structures in the Pager.aSavepoint[] array and set both ** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal @@ -42968,6 +45124,20 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +#ifdef SQLITE_HAS_CODEC +/* +** Make sure the number of reserved bits is the same in the destination +** pager as it is in the source. This comes up when a VACUUM changes the +** number of reserved bits to the "optimal" amount. +*/ +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ + if( pDest->nReserve!=pSrc->nReserve ){ + pDest->nReserve = pSrc->nReserve; + pagerReportSize(pDest); + } +} +#endif + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -43070,7 +45240,7 @@ static int pager_playback_one_page( } } - /* If this page has already been played by before during the current + /* If this page has already been played back before during the current ** rollback, then don't bother to play it back again. */ if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ @@ -43765,7 +45935,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ ** ** For an encrypted database, the situation is more complex: bytes ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** white noise equaling 16 bytes of 0xff is vanishingly small so ** we should still be ok. */ memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); @@ -43899,9 +46069,7 @@ static int pagerWalFrames( ){ int rc; /* Return code */ int nList; /* Number of pages in pList */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ -#endif assert( pPager->pWal ); assert( pList ); @@ -43918,7 +46086,6 @@ static int pagerWalFrames( ** any pages with page numbers greater than nTruncate into the WAL file. ** They will never be read by any client. So remove them from the pDirty ** list here. */ - PgHdr *p; PgHdr **ppNext = &pList; nList = 0; for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ @@ -43938,7 +46105,6 @@ static int pagerWalFrames( pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ - PgHdr *p; for(p=pList; p; p=p->pDirty){ sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } @@ -44008,11 +46174,10 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ assert( pPager->eLock>=SHARED_LOCK ); nPage = sqlite3WalDbsize(pPager->pWal); - /* If the database size was not available from the WAL sub-system, - ** determine it based on the size of the database file. If the size - ** of the database file is not an integer multiple of the page-size, - ** round down to the nearest page. Except, any file larger than 0 - ** bytes in size is considered to contain at least one page. + /* If the number of pages in the database is not available from the + ** WAL sub-system, determine the page counte based on the size of + ** the database file. If the size of the database file is not an + ** integer multiple of the page-size, round up the result. */ if( nPage==0 ){ i64 n = 0; /* Size of db file in bytes */ @@ -44753,7 +46918,7 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; @@ -45177,8 +47342,6 @@ static int openSubJournal(Pager *pPager){ /* ** Append a record of the current state of page pPg to the sub-journal. -** It is the callers responsibility to use subjRequiresPage() to check -** that it is really required before calling this function. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. @@ -45225,6 +47388,13 @@ static int subjournalPage(PgHdr *pPg){ } return rc; } +static int subjournalPageIfRequired(PgHdr *pPg){ + if( subjRequiresPage(pPg) ){ + return subjournalPage(pPg); + }else{ + return SQLITE_OK; + } +} /* ** This function is called by the pcache layer when it has reached some @@ -45282,9 +47452,7 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - if( subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); - } + rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } @@ -45297,39 +47465,6 @@ static int pagerStress(void *p, PgHdr *pPg){ rc = syncJournal(pPager, 1); } - /* If the page number of this page is larger than the current size of - ** the database image, it may need to be written to the sub-journal. - ** This is because the call to pager_write_pagelist() below will not - ** actually write data to the file in this case. - ** - ** Consider the following sequence of events: - ** - ** BEGIN; - ** - ** - ** SAVEPOINT sp; - ** - ** pagerStress(page X) - ** ROLLBACK TO sp; - ** - ** If (X>Y), then when pagerStress is called page X will not be written - ** out to the database file, but will be dropped from the cache. Then, - ** following the "ROLLBACK TO sp" statement, reading page X will read - ** data from the database file. This will be the copy of page X as it - ** was when the transaction started, not as it was when "SAVEPOINT sp" - ** was executed. - ** - ** The solution is to write the current data for page X into the - ** sub-journal file now (if it is not already there), so that it will - ** be restored to its current value when the "ROLLBACK TO sp" is - ** executed. - */ - if( NEVER( - rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) - ) ){ - rc = subjournalPage(pPg); - } - /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -45585,7 +47720,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ - pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -45604,7 +47739,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( nExtra<1000 ); nExtra = ROUND8(nExtra); rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } /* If an error occurred above, free the Pager structure and close the file. @@ -45984,18 +48119,14 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && ( - pPager->pBackup - || sqlite3PcachePagecount(pPager->pPCache)>0 - || USEFETCH(pPager) - )){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The hasHeldSharedLock flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. ** - ** Database changes is detected by looking at 15 bytes beginning + ** Database changes are detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when @@ -46061,6 +48192,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -46144,20 +48276,25 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -46202,9 +48339,14 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( if( pBase==0 ){ rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + pPg = *ppPage = 0; + rc = SQLITE_NOMEM; + goto pager_acquire_err; + } } pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); - if( pPg==0 ) rc = SQLITE_NOMEM; + assert( pPg!=0 ); } } @@ -46215,10 +48357,11 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( pPg = 0; goto pager_acquire_err; } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); + assert( pPg==(*ppPage) ); + assert( pPg->pgno==pgno ); + assert( pPg->pPager==pPager || pPg->pPager==0 ); - if( (*ppPage)->pPager && !noContent ){ + if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); @@ -46229,7 +48372,6 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( /* The pager cache has created a new page. Its content needs to ** be initialized. */ - pPg = *ppPage; pPg->pPager = pPager; /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page @@ -46307,6 +48449,8 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasHeldSharedLock ); + if( pPage==0 ) return 0; return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } @@ -46510,6 +48654,59 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory return rc; } +/* +** Write page pPg onto the end of the rollback journal. +*/ +static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc; + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + return rc; +} + /* ** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into @@ -46520,7 +48717,6 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - int inJournal; /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. @@ -46533,7 +48729,6 @@ static int pager_write(PgHdr *pPg){ assert( assert_pager_state(pPager) ); assert( pPager->errCode==0 ); assert( pPager->readOnly==0 ); - CHECK_PAGE(pPg); /* The journal file needs to be opened. Higher level routines have already @@ -46552,91 +48747,48 @@ static int pager_write(PgHdr *pPg){ assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); assert( assert_pager_state(pPager) ); - /* Mark the page as dirty. If the page has already been written - ** to the journal then we can return right away. - */ + /* Mark the page that is about to be modified as dirty. */ sqlite3PcacheMakeDirty(pPg); - inJournal = pageInJournal(pPager, pPg); - if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){ - assert( !pagerUseWal(pPager) ); - }else{ - - /* The transaction journal now exists and we have a RESERVED or an - ** EXCLUSIVE lock on the main database file. Write the current page to - ** the transaction journal if it is not there already. - */ - if( !inJournal && !pagerUseWal(pPager) ){ - assert( pagerUseWal(pPager)==0 ); - if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ - u32 cksum; - char *pData2; - i64 iOff = pPager->journalOff; - - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - - assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); - cksum = pager_cksum(pPager, (u8*)pData2); - - /* Even if an IO or diskfull error occurs while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - pPg->flags |= PGHDR_NEED_SYNC; - - rc = write32bits(pPager->jfd, iOff, pPg->pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); - if( rc!=SQLITE_OK ) return rc; - rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); - if( rc!=SQLITE_OK ) return rc; - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - - pPager->journalOff += 8 + pPager->pageSize; - pPager->nRec++; - assert( pPager->pInJournal!=0 ); - rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - rc |= addToSavepointBitvecs(pPager, pPg->pgno); - if( rc!=SQLITE_OK ){ - assert( rc==SQLITE_NOMEM ); - return rc; - } - }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD ){ - pPg->flags |= PGHDR_NEED_SYNC; - } - PAGERTRACE(("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + /* If a rollback journal is in use, them make sure the page that is about + ** to change is in the rollback journal, or if the page is a new page off + ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + */ + assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); + if( pPager->pInJournal!=0 + && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 + ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize ){ + rc = pagerAddPageToRollbackJournal(pPg); + if( rc!=SQLITE_OK ){ + return rc; } - } - - /* If the statement journal is open and the page is not in it, - ** then write the current page to the statement journal. Note that - ** the statement journal format differs from the standard journal format - ** in that it omits the checksums and the header. - */ - if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } } - /* Update the database size and return. + /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list + ** and before writing the page into the rollback journal. Wait until now, + ** after the page has been successfully journalled, before setting the + ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. + */ + pPg->flags |= PGHDR_WRITEABLE; + + /* If the statement journal is open and the page is not in it, + ** then write the page into the statement journal. */ + if( pPager->nSavepoint>0 ){ + rc = subjournalPageIfRequired(pPg); + } + + /* Update the database size and return. */ if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -46651,17 +48803,17 @@ static int pager_write(PgHdr *pPg){ ** a write. ** ** Usually, the sector size is less than or equal to the page size, in which -** case pages can be individually written. This routine only runs in the exceptional -** case where the page size is smaller than the sector size. +** case pages can be individually written. This routine only runs in the +** exceptional case where the page size is smaller than the sector size. */ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow @@ -46749,11 +48901,15 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ ** as appropriate. Otherwise, SQLITE_OK. */ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); - assert( pPg->pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPg->pPager->eState!=PAGER_ERROR ); - assert( assert_pager_state(pPg->pPager) ); - if( pPg->pPager->sectorSize > (u32)pPg->pPager->pageSize ){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); + return SQLITE_OK; + }else if( pPager->sectorSize > (u32)pPager->pageSize ){ return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -46767,7 +48923,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ */ #ifndef NDEBUG SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->flags&PGHDR_DIRTY; + return pPg->flags & PGHDR_WRITEABLE; } #endif @@ -46791,6 +48947,7 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; + pPg->flags &= ~PGHDR_WRITEABLE; pager_set_pagehash(pPg); } } @@ -47173,6 +49330,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + pPager->iDataVersion++; rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -47256,12 +49414,14 @@ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ return pPager->readOnly; } +#ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); } +#endif /* ** Return the approximate number of bytes of memory currently @@ -47344,54 +49504,62 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ +static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); + assert( nSavepoint>nCurrent && pPager->useJournal ); - if( nSavepoint>nCurrent && pPager->useJournal ){ - int ii; /* Iterator variable */ - PagerSavepoint *aNew; /* New Pager.aSavepoint array */ + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; + } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; - /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a - ** malloc failure occurs while populating it in the for(...) loop below. - */ - aNew = (PagerSavepoint *)sqlite3Realloc( - pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint - ); - if( !aNew ){ + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } - memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); - pPager->aSavepoint = aNew; - - /* Populate the PagerSavepoint structures just allocated. */ - for(ii=nCurrent; iidbSize; - if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ - aNew[ii].iOffset = pPager->journalOff; - }else{ - aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); - } - aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); - if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; - } - if( pagerUseWal(pPager) ){ - sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); - } - pPager->nSavepoint = ii+1; + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); } - assert( pPager->nSavepoint==nSavepoint ); - assertTruncateConstraint(pPager); + pPager->nSavepoint = ii+1; } - + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); return rc; } +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + return pagerOpenSavepoint(pPager, nSavepoint); + }else{ + return SQLITE_OK; + } +} + /* ** This function is called to rollback or release (commit) a savepoint. @@ -47622,9 +49790,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** one or more savepoint bitvecs. This is the reason this function ** may return SQLITE_NOMEM. */ - if( pPg->flags&PGHDR_DIRTY - && subjRequiresPage(pPg) - && SQLITE_OK!=(rc = subjournalPage(pPg)) + if( (pPg->flags & PGHDR_DIRTY)!=0 + && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) ){ return rc; } @@ -47713,6 +49880,18 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } #endif +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ + assert( pPg->pgno!=iNew ); + pPg->flags = flags; + sqlite3PcacheMove(pPg, iNew); +} + /* ** Return a pointer to the data for the specified page. */ @@ -47858,6 +50037,8 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } + }else if( eMode==PAGER_JOURNALMODE_OFF ){ + sqlite3OsClose(pPager->jfd); } } @@ -47929,7 +50110,8 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); @@ -48111,6 +50293,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ /************** End of pager.c ***********************************************/ @@ -48359,6 +50542,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ */ #ifndef SQLITE_OMIT_WAL +/* #include "wal.h" */ /* ** Trace output macros @@ -48544,6 +50728,7 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG @@ -48638,7 +50823,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ if( pWal->nWiData<=iPage ){ int nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); + apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; return SQLITE_NOMEM; @@ -48764,9 +50949,9 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); - memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); - memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); } /* @@ -48904,9 +51089,10 @@ static void walUnlockShared(Wal *pWal, int lockIdx){ SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } -static int walLockExclusive(Wal *pWal, int lockIdx, int n){ +static int walLockExclusive(Wal *pWal, int lockIdx, int n, int fBlock){ int rc; if( pWal->exclusiveMode ) return SQLITE_OK; + if( fBlock ) sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_WAL_BLOCK, 0); rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, @@ -49067,13 +51253,13 @@ static void walCleanupHash(Wal *pWal){ ** via the hash table even after the cleanup. */ if( iLimit ){ - int i; /* Loop counter */ + int j; /* Loop counter */ int iKey; /* Hash key */ - for(i=1; i<=iLimit; i++){ - for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==i ) break; + for(j=1; j<=iLimit; j++){ + for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==j ) break; } - assert( aHash[iKey]==i ); + assert( aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -49192,7 +51378,7 @@ static int walIndexRecover(Wal *pWal){ assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, nLock, 0); if( rc ){ return rc; } @@ -49262,7 +51448,7 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc(szFrame); + aFrame = (u8 *)sqlite3_malloc64(szFrame); if( !aFrame ){ rc = SQLITE_NOMEM; goto recovery_error; @@ -49575,7 +51761,7 @@ static void walMergesort( int nMerge = 0; /* Number of elements in list aMerge */ ht_slot *aMerge = 0; /* List to be merged */ int iList; /* Index into input list */ - int iSub = 0; /* Index into aSub array */ + u32 iSub = 0; /* Index into aSub array */ struct Sublist aSub[13]; /* Array of sub-lists */ memset(aSub, 0, sizeof(aSub)); @@ -49586,7 +51772,9 @@ static void walMergesort( nMerge = 1; aMerge = &aList[iList]; for(iSub=0; iList & (1<aList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -49597,7 +51785,9 @@ static void walMergesort( for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -49620,7 +51810,7 @@ static void walMergesort( ** Free an iterator allocated by walIteratorInit(). */ static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); + sqlite3_free(p); } /* @@ -49655,7 +51845,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte); if( !p ){ return SQLITE_NOMEM; } @@ -49665,7 +51855,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( + aTmp = (ht_slot *)sqlite3_malloc64( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ @@ -49702,7 +51892,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ p->aSegment[i].aPgno = (u32 *)aPgno; } } - sqlite3ScratchFree(aTmp); + sqlite3_free(aTmp); if( rc!=SQLITE_OK ){ walIteratorFree(p); @@ -49726,7 +51916,7 @@ static int walBusyLock( ){ int rc; do { - rc = walLockExclusive(pWal, lockIdx, n); + rc = walLockExclusive(pWal, lockIdx, n, 0); }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); return rc; } @@ -49739,6 +51929,38 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -49773,12 +51995,12 @@ static int walPagesize(Wal *pWal){ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ @@ -49787,123 +52009,154 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); pInfo = walCkptInfo(pWal); - if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + if( pInfo->nBackfillhdr.mxFrame ){ - /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pIter ); + /* Allocate the iterator */ + rc = walIteratorInit(pWal, &pIter); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pIter ); - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); - /* Compute in mxSafeFrame the index of the last frame of the WAL that is - ** safe to write into the database. Frames beyond mxSafeFrame might - ** overwrite database pages that are in use by active readers and thus - ** cannot be backfilled from the WAL. - */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - xBusy = 0; - }else{ - goto walcheckpoint_out; + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. + */ + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; + } } } - } - - if( pInfo->nBackfillnBackfill; - /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } + if( pInfo->nBackfillnBackfill; - /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + /* Sync the WAL to disk */ + if( sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - } + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } - /* Iterate through the contents of the WAL, copying data to the db file. */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + /* Iterate through the contents of the WAL, copying data to the db file */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + continue; } + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; } + + /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK && sync_flags ){ + rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + } + } + if( rc==SQLITE_OK ){ + pInfo->nBackfill = mxSafeFrame; + } } - } - /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); - } + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } - if( rc==SQLITE_BUSY ){ - /* Reset the return code so as not to report a checkpoint failure - ** just because there are active readers. */ - rc = SQLITE_OK; + if( rc==SQLITE_BUSY ){ + /* Reset the return code so as not to report a checkpoint failure + ** just because there are active readers. */ + rc = SQLITE_OK; + } } - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ - assert( mxSafeFrame==pWal->hdr.mxFrame ); + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -50104,7 +52357,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } - }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 1)) ){ pWal->writeLock = 1; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); @@ -50310,7 +52563,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ && (mxReadMarkhdr.mxFrame || mxI==0) ){ for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; mxI = i; @@ -50344,12 +52597,27 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry ** instead. ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ + pWal->minFrame = pInfo->nBackfill+1; walShmBarrier(pWal); if( pInfo->aReadMark[mxI]!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) @@ -50420,6 +52688,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -50460,7 +52729,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -50475,8 +52745,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - /* assert( iFrame>iRead ); -- not true if there is corruption */ + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ @@ -50492,7 +52762,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ + assert( pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -50566,7 +52837,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); if( rc ){ return rc; } @@ -50689,7 +52960,6 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -50712,7 +52982,7 @@ static int walRestartLog(Wal *pWal){ if( pInfo->nBackfill>0 ){ u32 salt1; sqlite3_randomness(4, &salt1); - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1, 0); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no ** readers are currently using the WAL), then the transactions @@ -50722,20 +52992,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; @@ -51023,7 +53281,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( */ SQLITE_PRIVATE int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -51035,29 +53293,42 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1, 0); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); @@ -51065,6 +53336,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } @@ -51082,7 +53354,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } /* If no error occurred, set the output variables. */ @@ -51444,6 +53716,7 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ ** 4 Number of leaf pointers on this page ** * zero or more pages numbers of leaves */ +/* #include "sqliteInt.h" */ /* The following value is the maximum cell size assuming a maximum page @@ -51461,6 +53734,7 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; +typedef struct CellInfo CellInfo; /* ** This is a magic string that appears at the beginning of every @@ -51510,6 +53784,7 @@ struct MemPage { u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ + u8 bBusy; /* Prevent endless loops on corrupt database files */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ @@ -51523,7 +53798,10 @@ struct MemPage { u8 *aData; /* Pointer to disk image of the page data */ u8 *aDataEnd; /* One byte past the end of usable data */ u8 *aCellIdx; /* The cell index area */ + u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ + u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ + void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ Pgno pgno; /* Page number for this page */ }; @@ -51579,8 +53857,10 @@ struct Btree { u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ u8 sharable; /* True if we can share pBt with another db */ u8 locked; /* True if db currently has pBt locked */ + u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ + u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ #ifndef SQLITE_OMIT_SHARED_CACHE @@ -51647,6 +53927,9 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ +#ifdef SQLITE_HAS_CODEC + u8 optimalReserve; /* Desired amount of reserved space per page */ +#endif u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ @@ -51685,7 +53968,6 @@ struct BtShared { ** about a cell. The parseCellPtr() function fills in this structure ** based on information extract from the raw disk page. */ -typedef struct CellInfo CellInfo; struct CellInfo { i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ u8 *pPayload; /* Pointer to the start of payload */ @@ -51728,8 +54010,7 @@ struct CellInfo { struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + BtCursor *pNext; /* Forms a linked list of all cursors */ Pgno *aOverflow; /* Cache of overflow page locations */ CellInfo info; /* A parse of the cell we are pointing at */ i64 nKey; /* Size of pKey, or last integer key */ @@ -51739,9 +54020,16 @@ struct BtCursor { int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerAcquire() */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ - u8 hints; /* As configured by CursorSetHints() */ - i16 iPage; /* Index of current page in apPage */ + u8 hints; /* As configured by CursorSetHints() */ + /* All fields above are zeroed when the cursor is allocated. See + ** sqlite3BtreeCursorZero(). Fields that follow must be manually + ** initialized. */ + i8 iPage; /* Index of current page in apPage */ + u8 curIntKey; /* Value of apPage[0]->intKey */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + void *padding1; /* Make object size a multiple of 16 */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; @@ -51754,6 +54042,7 @@ struct BtCursor { #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ +#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ /* ** Potential values for BtCursor.eState. @@ -51896,6 +54185,7 @@ struct IntegrityCk { const char *zPfx; /* Error message prefix */ int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ }; /* @@ -51906,6 +54196,23 @@ struct IntegrityCk { #define get4byte sqlite3Get4byte #define put4byte sqlite3Put4byte +/* +** get2byteAligned(), unlike get2byte(), requires that its argument point to a +** two-byte aligned address. get2bytea() is only used for accessing the +** cell addresses in a btree header. +*/ +#if SQLITE_BYTEORDER==4321 +# define get2byteAligned(x) (*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && GCC_VERSION>=4008000 +# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 +# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) +#else +# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) +#endif + /************** End of btreeInt.h ********************************************/ /************** Continuing where we left off in btmutex.c ********************/ #ifndef SQLITE_OMIT_SHARED_CACHE @@ -52033,6 +54340,7 @@ static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ ** Exit the recursive mutex on a Btree. */ SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ + assert( sqlite3_mutex_held(p->db->mutex) ); if( p->sharable ){ assert( p->wantToLock>0 ); p->wantToLock--; @@ -52208,6 +54516,7 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ +/* #include "btreeInt.h" */ /* ** The header string that appears at the beginning of every @@ -52280,7 +54589,7 @@ static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; ** The shared cache setting effects only future calls to ** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). */ -SQLITE_API int sqlite3_enable_shared_cache(int enable){ +SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int enable){ sqlite3GlobalConfig.sharedCacheEnabled = enable; return SQLITE_OK; } @@ -52369,6 +54678,12 @@ static int hasSharedCacheTableLock( for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ Index *pIdx = (Index *)sqliteHashData(p); if( pIdx->tnum==(int)iRoot ){ + if( iTab ){ + /* Two or more indexes share the same root page. There must + ** be imposter tables. So just return true. The assert is not + ** useful in that case. */ + return 1; + } iTab = pIdx->pTable->tnum; } } @@ -52678,13 +54993,15 @@ static void invalidateIncrblobCursors( int isClearTable /* True if all rows are being deleted */ ){ BtCursor *p; - BtShared *pBt = pBtree->pBt; + if( pBtree->hasIncrblobCur==0 ) return; assert( sqlite3BtreeHoldsMutex(pBtree) ); - for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 - && (isClearTable || p->info.nKey==iRow) - ){ - p->eState = CURSOR_INVALID; + pBtree->hasIncrblobCur = 0; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 ){ + pBtree->hasIncrblobCur = 1; + if( isClearTable || p->info.nKey==iRow ){ + p->eState = CURSOR_INVALID; + } } } } @@ -52777,17 +55094,21 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } - /* -** Save the current cursor position in the variables BtCursor.nKey -** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. ** -** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. */ -static int saveCursorPosition(BtCursor *pCur){ +static int saveCursorKey(BtCursor *pCur){ int rc; - assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); @@ -52799,9 +55120,8 @@ static int saveCursorPosition(BtCursor *pCur){ ** stores the integer key in pCur->nKey. In this case this value is ** all that is required. Otherwise, if pCur is not open on an intKey ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->apPage[0]->intKey ){ + ** data. */ + if( 0==pCur->curIntKey ){ void *pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); @@ -52814,14 +55134,37 @@ static int saveCursorPosition(BtCursor *pCur){ rc = SQLITE_NOMEM; } } - assert( !pCur->apPage[0]->intKey || !pCur->pKey ); + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; + } + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - invalidateOverflowCache(pCur); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); return rc; } @@ -52836,6 +55179,15 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** routine is called just before cursor pExcept is used to modify the ** table, for example in BtreeDelete() or BtreeInsert(). ** +** If there are two or more cursors on the same btree, then all such +** cursors should have their BTCF_Multiple flag set. The btreeCursor() +** routine enforces that rule. This routine only needs to be called in +** the uncommon case when pExpect has the BTCF_Multiple flag set. +** +** If pExpect!=NULL and if no other cursors are found on the same root-page, +** then the BTCF_Multiple flag on pExpect is cleared, to avoid another +** pointless call to this routine. +** ** Implementation note: This routine merely checks to see if any cursors ** need to be saved. It calls out to saveCursorsOnList() in the (unusual) ** event that cursors are in need to being saved. @@ -52847,7 +55199,9 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ for(p=pBt->pCursor; p; p=p->pNext){ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; } - return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK; + if( p ) return saveCursorsOnList(p, iRoot, pExcept); + if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; + return SQLITE_OK; } /* This helper routine to saveAllCursors does the actual work of saving @@ -52862,7 +55216,7 @@ static int SQLITE_NOINLINE saveCursorsOnList( ){ do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ - if( p->eState==CURSOR_VALID ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ int rc = saveCursorPosition(p); if( SQLITE_OK!=rc ){ return rc; @@ -52934,17 +55288,19 @@ static int btreeMoveto( */ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; + int skipNext; assert( cursorHoldsMutex(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext); + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); + pCur->skipNext |= skipNext; if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ pCur->eState = CURSOR_SKIPNEXT; } @@ -52996,9 +55352,10 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) *pDifferentRow = 1; return rc; } - if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ + if( pCur->eState!=CURSOR_VALID ){ *pDifferentRow = 1; }else{ + assert( pCur->skipNext==0 ); *pDifferentRow = 0; } return SQLITE_OK; @@ -53132,39 +55489,88 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** the page, 1 means the second cell, and so forth) return a pointer ** to the cell content. ** +** findCellPastPtr() does the same except it skips past the initial +** 4-byte child pointer found on interior pages, if there is one. +** ** This routine works only for pages that do not contain overflow cells. */ #define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) -#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) +#define findCellPastPtr(P,I) \ + ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) /* -** This a more complex version of findCell() that works for -** pages that do contain overflow cells. +** This is common tail processing for btreeParseCellPtr() and +** btreeParseCellPtrIndex() for the case when the cell does not fit entirely +** on a single B-tree page. Make necessary adjustments to the CellInfo +** structure. */ -static u8 *findOverflowCell(MemPage *pPage, int iCell){ - int i; - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - for(i=pPage->nOverflow-1; i>=0; i--){ - int k; - k = pPage->aiOvfl[i]; - if( k<=iCell ){ - if( k==iCell ){ - return pPage->apOvfl[i]; - } - iCell--; - } +static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; } - return findCell(pPage, iCell); + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); + pInfo->nSize = pInfo->iOverflow + 4; } /* -** Parse a cell content block and fill in the CellInfo structure. There -** are two versions of this function. btreeParseCell() takes a -** cell index as the second argument and btreeParseCellPtr() -** takes a pointer to the body of the cell as its second argument. +** The following routines are implementations of the MemPage.xParseCell() +** method. +** +** Parse a cell content block and fill in the CellInfo structure. +** +** btreeParseCellPtr() => table btree leaf nodes +** btreeParseCellNoPayload() => table btree internal nodes +** btreeParseCellPtrIndex() => index btree nodes +** +** There is also a wrapper function btreeParseCell() that works for +** all MemPage types and that references the cell by index rather than +** by pointer. */ +static void btreeParseCellPtrNoPayload( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 ); + assert( pPage->noPayload ); + assert( pPage->childPtrSize==4 ); +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER(pPage); +#endif + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; +} static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ @@ -53172,26 +55578,93 @@ static void btreeParseCellPtr( ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ + u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - if( pPage->intKeyLeaf ){ - assert( pPage->childPtrSize==0 ); - pIter = pCell + getVarint32(pCell, nPayload); - pIter += getVarint(pIter, (u64*)&pInfo->nKey); - }else if( pPage->noPayload ){ - assert( pPage->childPtrSize==4 ); - pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); - pInfo->nPayload = 0; - pInfo->nLocal = 0; + assert( pPage->intKeyLeaf || pPage->noPayload ); + assert( pPage->noPayload==0 ); + assert( pPage->intKeyLeaf ); + assert( pPage->childPtrSize==0 ); + pIter = pCell; + + /* The next block of code is equivalent to: + ** + ** pIter += getVarint32(pIter, nPayload); + ** + ** The code is inlined to avoid a function call. + */ + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( (*pIter)>=0x80 && pIternKey); + ** + ** The code is inlined to avoid a function call. + */ + iKey = *pIter; + if( iKey>=0x80 ){ + u8 *pEnd = &pIter[7]; + iKey &= 0x7f; + while(1){ + iKey = (iKey<<7) | (*++pIter & 0x7f); + if( (*pIter)<0x80 ) break; + if( pIter>=pEnd ){ + iKey = (iKey<<8) | *++pIter; + break; + } + } + } + pIter++; + + pInfo->nKey = *(i64*)&iKey; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; - pInfo->pPayload = 0; - return; }else{ - pIter = pCell + pPage->childPtrSize; - pIter += getVarint32(pIter, nPayload); - pInfo->nKey = nPayload; + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } +} +static void btreeParseCellPtrIndex( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf==0 ); + assert( pPage->noPayload==0 ); + pIter = pCell + pPage->childPtrSize; + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIternKey = nPayload; pInfo->nPayload = nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); @@ -53205,31 +55678,7 @@ static void btreeParseCellPtr( pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; }else{ - /* If the payload will not fit completely on the local page, we have - ** to decide how much to store locally and how much to spill onto - ** overflow pages. The strategy is to minimize the amount of unused - ** space on overflow pages while keeping the amount of local storage - ** in between minLocal and maxLocal. - ** - ** Warning: changing the way overflow payload is distributed in any - ** way will result in an incompatible file format. - */ - int minLocal; /* Minimum amount of payload held locally */ - int maxLocal; /* Maximum amount of payload held locally */ - int surplus; /* Overflow payload available for local storage */ - - minLocal = pPage->minLocal; - maxLocal = pPage->maxLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); - testcase( surplus==maxLocal ); - testcase( surplus==maxLocal+1 ); - if( surplus <= maxLocal ){ - pInfo->nLocal = (u16)surplus; - }else{ - pInfo->nLocal = (u16)minLocal; - } - pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); - pInfo->nSize = pInfo->iOverflow + 4; + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } } static void btreeParseCell( @@ -53237,14 +55686,20 @@ static void btreeParseCell( int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); + pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); } /* +** The following routines are implementations of the MemPage.xCellSize +** method. +** ** Compute the total number of bytes that a Cell needs in the cell ** data area of the btree-page. The return number includes the cell ** data header and the local payload, but not any overflow page or ** the space used by the cell pointer. +** +** cellSizePtrNoPayload() => table internal nodes +** cellSizePtr() => all index nodes & table leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ @@ -53257,18 +55712,13 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; - btreeParseCellPtr(pPage, pCell, &debuginfo); + pPage->xParseCell(pPage, pCell, &debuginfo); #endif - if( pPage->noPayload ){ - pEnd = &pIter[9]; - while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); - return (u16)(pIter - pCell); - } + assert( pPage->noPayload==0 ); nSize = *pIter; if( nSize>=0x80 ){ - pEnd = &pIter[9]; + pEnd = &pIter[8]; nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); @@ -53300,12 +55750,34 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } +static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#else + UNUSED_PARAMETER(pPage); +#endif + + assert( pPage->childPtrSize==4 ); + pEnd = pIter + 9; + while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); } #endif @@ -53319,7 +55791,7 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); if( info.iOverflow ){ Pgno ovfl = get4byte(&pCell[info.iOverflow]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); @@ -53333,6 +55805,11 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ ** end of the page and all free space is collected into one ** big FreeBlk that occurs in between the header and cell ** pointer array and the cell content area. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ @@ -53345,6 +55822,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -53354,15 +55832,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -53372,31 +55848,31 @@ static int defragmentPage(MemPage *pPage){ pc = get2byte(pAddr); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); -#if !defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) /* These conditions have already been verified in btreeInitPage() - ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined + ** if PRAGMA cell_size_check=ON. */ if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - if( cbrkusableSize ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -53411,6 +55887,70 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** +** Slots on the free list that are between 1 and 3 bytes larger than nByte +** will be ignored if adding the extra space to the fragmentation count +** causes the fragmentation count to exceed 60. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr = hdr + 1; + int pc = get2byte(&aData[iAddr]); + int x; + int usableSize = pPg->pBt->usableSize; + + assert( pc>0 ); + do{ + int size; /* Size of the free slot */ + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ + if( pc>usableSize-4 || pc=0 ){ + testcase( x==4 ); + testcase( x==3 ); + if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ + *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ + if( aData[hdr+7]>57 ) return 0; + + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + iAddr = pc; + pc = get2byte(&aData[pc]); + }while( pc ); + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -53428,9 +55968,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -53438,15 +55977,20 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; assert( gap<=65536 ); + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ top = get2byte(&data[hdr+5]); + assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ if( gap>top ){ - if( top==0 ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ return SQLITE_CORRUPT_BKPT; @@ -53460,34 +56004,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( data[hdr+7]>=60 ) goto defragment_page; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( pSpace ){ + assert( pSpace>=data && (pSpace - data)<65536 ); + *pIdx = (int)(pSpace - data); + return SQLITE_OK; + }else if( rc ){ + return rc; } } @@ -53496,8 +56020,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ -defragment_page: - testcase( pPage->nCell==0 ); + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -53543,8 +56066,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ assert( iStart<=iLast ); @@ -53572,7 +56095,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none - ** iPtr: The address of a pointer iFreeBlk + ** iPtr: The address of a pointer to iFreeBlk ** ** Check to see if iFreeBlk should be coalesced onto the end of iStart. */ @@ -53580,6 +56103,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT; iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); } @@ -53637,20 +56161,44 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); flagByte &= ~PTF_LEAF; pPage->childPtrSize = 4-4*pPage->leaf; + pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; - pPage->intKeyLeaf = pPage->leaf; - pPage->noPayload = !pPage->leaf; + if( pPage->leaf ){ + pPage->intKeyLeaf = 1; + pPage->noPayload = 0; + pPage->xParseCell = btreeParseCellPtr; + }else{ + pPage->intKeyLeaf = 0; + pPage->noPayload = 1; + pPage->xCellSize = cellSizePtrNoPayload; + pPage->xParseCell = btreeParseCellPtrNoPayload; + } pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior + ** index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf + ** index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; pPage->noPayload = 0; + pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ return SQLITE_CORRUPT_BKPT; } pPage->max1bytePayload = pBt->max1bytePayload; @@ -53669,6 +56217,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ static int btreeInitPage(MemPage *pPage){ assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); @@ -53690,21 +56239,34 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + pPage->aDataOfst = &data[pPage->childPtrSize]; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); /* A malformed database page might cause us to read past the end ** of page when parsing a cell. @@ -53715,20 +56277,19 @@ static int btreeInitPage(MemPage *pPage){ */ iCellFirst = cellOffset + 2*pPage->nCell; iCellLast = usableSize - 4; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - { + if( pBt->db->flags & SQLITE_CellSizeCk ){ int i; /* Index into the cell pointer array */ int sz; /* Size of a cell */ if( !pPage->leaf ) iCellLast--; for(i=0; inCell; i++){ - pc = get2byte(&data[cellOffset+i*2]); + pc = get2byteAligned(&data[cellOffset+i*2]); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } - sz = cellSizePtr(pPage, &data[pc]); + sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ return SQLITE_CORRUPT_BKPT; @@ -53736,15 +56297,21 @@ static int btreeInitPage(MemPage *pPage){ } if( !pPage->leaf ) iCellLast++; } -#endif - /* Compute the total free space on the page */ + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ while( pc>0 ){ u16 next, size; if( pciCellLast ){ - /* Start of free block is off the page */ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + ** + ** Or, the freeblock is off the end of the page + */ return SQLITE_CORRUPT_BKPT; } next = get2byte(&data[pc]); @@ -53802,6 +56369,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->cellOffset = first; pPage->aDataEnd = &data[pBt->usableSize]; pPage->aCellIdx = &data[first]; + pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -53820,16 +56388,16 @@ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ pPage->pDbPage = pDbPage; pPage->pBt = pBt; pPage->pgno = pgno; - pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + pPage->hdrOffset = pgno==1 ? 100 : 0; return pPage; } /* ** Get a page from the pager. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. +** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). ** -** If the noContent flag is set, it means that we do not care about -** the content of the page at this time. So do not go to the disk +** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care +** about the content of the page at this time. So do not go to the disk ** to fetch the content. Just fill in the content with zeros for now. ** If in the future we call sqlite3PagerWrite() on this page, that ** means we have started to be concerned about content and the disk @@ -53881,35 +56449,62 @@ SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ } /* -** Get a page from the pager and initialize it. This routine is just a -** convenience wrapper around separate calls to btreeGetPage() and -** btreeInitPage(). +** Get a page from the pager and initialize it. +** +** If pCur!=0 then the page is being fetched as part of a moveToChild() +** call. Do additional sanity checking on the page in this case. +** And if the fetch fails, this routine must decrement pCur->iPage. +** +** The page is fetched as read-write unless pCur is not NULL and is +** a read-only cursor. ** -** If an error occurs, then the value *ppPage is set to is undefined. It +** If an error occurs, then *ppPage is undefined. It ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - int bReadonly /* PAGER_GET_READONLY or 0 */ + BtCursor *pCur, /* Cursor to receive the page, or NULL */ + int bReadOnly /* True for a read-only page */ ){ int rc; + DbPage *pDbPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( bReadonly==PAGER_GET_READONLY || bReadonly==0 ); + assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); + assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); + assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = btreeGetPage(pBt, pgno, ppPage, bReadonly); - if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ - rc = btreeInitPage(*ppPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } + goto getAndInitPage_error; + } + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ + goto getAndInitPage_error; + } + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + if( (*ppPage)->isInit==0 ){ + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + goto getAndInitPage_error; } } + /* If obtaining a child page for a cursor, we must verify that the page is + ** compatible with the root page. */ + if( pCur + && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) + ){ + rc = SQLITE_CORRUPT_BKPT; + releasePage(*ppPage); + goto getAndInitPage_error; + } + return SQLITE_OK; + +getAndInitPage_error: + if( pCur ) pCur->iPage--; testcase( pgno==0 ); assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; @@ -53919,18 +56514,49 @@ static int getAndInitPage( ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. */ +static void releasePageNotNull(MemPage *pPage){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); +} static void releasePage(MemPage *pPage){ - if( pPage ){ - assert( pPage->aData ); - assert( pPage->pBt ); - assert( pPage->pDbPage!=0 ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - sqlite3PagerUnrefNotNull(pPage->pDbPage); + if( pPage ) releasePageNotNull(pPage); +} + +/* +** Get an unused page. +** +** This works just like btreeGetPage() with the addition: +** +** * If the page is already in use for some other purpose, immediately +** release it and return an SQLITE_CURRUPT error. +** * Make sure the isInit flag is clear +*/ +static int btreeGetUnusedPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc = btreeGetPage(pBt, pgno, ppPage, flags); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; } + return rc; } + /* ** During a rollback, when the pager reloads information into the cache ** so that the cache is restored to its original state at the start of @@ -54053,16 +56679,18 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( */ if( isTempDb==0 && (isMemdb==0 || (vfsFlags&SQLITE_OPEN_URI)!=0) ){ if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFilename = sqlite3Strlen30(zFilename)+1; int nFullPathname = pVfs->mxPathname+1; - char *zFullPathname = sqlite3Malloc(nFullPathname); + char *zFullPathname = sqlite3Malloc(MAX(nFullPathname,nFilename)); MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); return SQLITE_NOMEM; } if( isMemdb ){ - memcpy(zFullPathname, zFilename, sqlite3Strlen30(zFilename)+1); + memcpy(zFullPathname, zFilename, nFilename); }else{ rc = sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); @@ -54119,8 +56747,8 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** the right size. This is to guard against size changes that result ** when compiling on a different architecture. */ - assert( sizeof(i64)==8 || sizeof(i64)==4 ); - assert( sizeof(u64)==8 || sizeof(u64)==4 ); + assert( sizeof(i64)==8 ); + assert( sizeof(u64)==8 ); assert( sizeof(u32)==4 ); assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); @@ -54150,6 +56778,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->btsFlags |= BTS_SECURE_DELETE; #endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -54168,6 +56799,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #endif nReserve = 0; }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ nReserve = zDbHeader[20]; pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -54501,6 +57135,9 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, BtShared *pBt = p->pBt; assert( nReserve>=-1 && nReserve<=255 ); sqlite3BtreeEnter(p); +#if SQLITE_HAS_CODEC + if( nReserve>pBt->optimalReserve ) pBt->optimalReserve = (u8)nReserve; +#endif if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; @@ -54512,7 +57149,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); - assert( !pBt->pPage1 && !pBt->pCursor ); + assert( !pBt->pCursor ); pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } @@ -54530,7 +57167,6 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) /* ** This function is similar to sqlite3BtreeGetReserve(), except that it ** may only be called if it is guaranteed that the b-tree mutex is already @@ -54543,25 +57179,33 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ ** database handle that owns *p, causing undefined behavior. */ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ + int n; assert( sqlite3_mutex_held(p->pBt->mutex) ); - return p->pBt->pageSize - p->pBt->usableSize; + n = p->pBt->pageSize - p->pBt->usableSize; + return n; } -#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* ** Return the number of bytes of space at the end of every page that ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. +** +** If SQLITE_HAS_MUTEX is defined then the number returned is the +** greater of the current reserved space and the maximum requested +** reserve space. */ -SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree *p){ +SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ int n; sqlite3BtreeEnter(p); - n = p->pBt->pageSize - p->pBt->usableSize; + n = sqlite3BtreeGetReserveNoMutex(p); +#ifdef SQLITE_HAS_CODEC + if( npBt->optimalReserve ) n = p->pBt->optimalReserve; +#endif sqlite3BtreeLeave(p); return n; } + /* ** Set the maximum page count for a database if mxPage is positive. ** No changes are made if mxPage is 0 or negative. @@ -54592,7 +57236,6 @@ SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ sqlite3BtreeLeave(p); return b; } -#endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ /* ** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' @@ -54677,6 +57320,9 @@ static int lockBtree(BtShared *pBt){ u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -54717,15 +57363,21 @@ static int lockBtree(BtShared *pBt){ } #endif - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** ** The original design allowed these amounts to vary, but as of ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 || pageSize>SQLITE_MAX_PAGE_SIZE || pageSize<=256 @@ -54733,6 +57385,13 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ usableSize = pageSize - page1[20]; if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size @@ -54753,6 +57412,9 @@ static int lockBtree(BtShared *pBt){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ if( usableSize<480 ){ goto page1_init_failed; } @@ -54837,7 +57499,7 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); pBt->pPage1 = 0; - releasePage(pPage1); + releasePageNotNull(pPage1); } } @@ -55142,15 +57804,17 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ u8 isInitOrig = pPage->isInit; int i; int nCell; + int rc; - btreeInitPage(pPage); + rc = btreeInitPage(pPage); + if( rc ) return rc; nCell = pPage->nCell; for(i=0; ixParseCell(pPage, pCell, &info); if( info.iOverflow && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage && iFrom==get4byte(&pCell[info.iOverflow]) @@ -55449,7 +58113,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); @@ -55633,6 +58297,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeLeave(p); return rc; } + p->iDataVersion--; /* Compensate for pPager->iDataVersion++; */ pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -55692,7 +58357,7 @@ SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int wr for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ - if( p->eState==CURSOR_VALID ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); @@ -55890,6 +58555,7 @@ static int btreeCursor( BtCursor *pCur /* Space for new cursor */ ){ BtShared *pBt = p->pBt; /* Shared b-tree handle */ + BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); assert( wrFlag==0 || wrFlag==1 ); @@ -55905,10 +58571,8 @@ static int btreeCursor( assert( p->inTrans>TRANS_NONE ); assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); assert( pBt->pPage1 && pBt->pPage1->aData ); + assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); - if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ - return SQLITE_READONLY; - } if( wrFlag ){ allocateTempSpace(pBt); if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; @@ -55927,10 +58591,16 @@ static int btreeCursor( pCur->pBt = pBt; assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); pCur->curFlags = wrFlag; - pCur->pNext = pBt->pCursor; - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur; + pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + /* If there are two or more cursors on the same btree, then all such + ** cursors *must* have the BTCF_Multiple flag set. */ + for(pX=pBt->pCursor; pX; pX=pX->pNext){ + if( pX->pgnoRoot==(Pgno)iTable ){ + pX->curFlags |= BTCF_Multiple; + pCur->curFlags |= BTCF_Multiple; + } } + pCur->pNext = pBt->pCursor; pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; return SQLITE_OK; @@ -55943,9 +58613,13 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( BtCursor *pCur /* Write new cursor here */ ){ int rc; - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); + if( iTable<1 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + } return rc; } @@ -55984,19 +58658,24 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); sqlite3BtreeClearCursor(pCur); - if( pCur->pPrev ){ - pCur->pPrev->pNext = pCur->pNext; - }else{ + assert( pBt->pCursor!=0 ); + if( pBt->pCursor==pCur ){ pBt->pCursor = pCur->pNext; - } - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur->pPrev; + }else{ + BtCursor *pPrev = pBt->pCursor; + do{ + if( pPrev->pNext==pCur ){ + pPrev->pNext = pCur->pNext; + break; + } + pPrev = pPrev->pNext; + }while( ALWAYS(pPrev) ); } for(i=0; i<=pCur->iPage; i++){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - sqlite3DbFree(pBtree->db, pCur->aOverflow); + sqlite3_free(pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -56010,13 +58689,6 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** ** BtCursor.info is a cache of the information in the current cell. ** Using this cache reduces the number of calls to btreeParseCell(). -** -** 2007-06-25: There is a bug in some versions of MSVC that cause the -** compiler to crash when getCellInfo() is implemented as a macro. -** But there is a measureable speed advantage to using the macro on gcc -** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing aggressive inlining.) So we use a real function -** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ @@ -56029,28 +58701,15 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ #else #define assertCellInfo(x) #endif -#ifdef _MSC_VER - /* Use a real function in MSVC to work around bugs in that compiler. */ - static void getCellInfo(BtCursor *pCur){ - if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->curFlags |= BTCF_ValidNKey; - }else{ - assertCellInfo(pCur); - } - } -#else /* if not _MSC_VER */ - /* Use a macro in all other compilers so that the function is inlined */ -#define getCellInfo(pCur) \ - if( pCur->info.nSize==0 ){ \ - int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->curFlags |= BTCF_ValidNKey; \ - }else{ \ - assertCellInfo(pCur); \ +static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + pCur->curFlags |= BTCF_ValidNKey; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + }else{ + assertCellInfo(pCur); } -#endif /* _MSC_VER */ +} #ifndef NDEBUG /* The next routine used only within assert() statements */ /* @@ -56098,6 +58757,8 @@ SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 ); + assert( pCur->iPageapPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); *pSize = pCur->info.nPayload; @@ -56291,6 +58952,7 @@ static int accessPayload( offset -= pCur->info.nLocal; } + if( rc==SQLITE_OK && amt>0 ){ const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ Pgno nextPage; @@ -56308,8 +58970,8 @@ static int accessPayload( if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ - Pgno *aNew = (Pgno*)sqlite3DbRealloc( - pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ rc = SQLITE_NOMEM; @@ -56356,6 +59018,7 @@ static int accessPayload( */ assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; }else{ @@ -56504,13 +59167,18 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ + u32 amt; assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorHoldsMutex(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); - *pAmt = pCur->info.nLocal; + assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadapPage[pCur->iPage]->aDataEnd ||CORRUPT_DB); + amt = (int)(pCur->apPage[pCur->iPage]->aDataEnd - pCur->info.pPayload); + if( pCur->info.nLocalinfo.nLocal; + *pAmt = amt; return (void*)pCur->info.pPayload; } @@ -56547,9 +59215,6 @@ SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - int rc; - int i = pCur->iPage; - MemPage *pNewPage; BtShared *pBt = pCur->pBt; assert( cursorHoldsMutex(pCur) ); @@ -56559,22 +59224,15 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage, - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); - if( rc ) return rc; - pCur->apPage[i+1] = pNewPage; - pCur->aiIdx[i+1] = 0; - pCur->iPage++; - pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } - return SQLITE_OK; + pCur->iPage++; + pCur->aiIdx[pCur->iPage] = 0; + return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], + pCur, pCur->curPagerFlags); } -#if 0 +#if SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -56583,6 +59241,8 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ ** the page. */ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + if( CORRUPT_DB ) return; /* The conditions tested below might not be true + ** in a corrupt database */ assert( iIdx<=pParent->nCell ); if( iIdx==pParent->nCell ){ assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); @@ -56607,25 +59267,15 @@ static void moveToParent(BtCursor *pCur){ assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); assert( pCur->apPage[pCur->iPage] ); - - /* UPDATE: It is actually possible for the condition tested by the assert - ** below to be untrue if the database file is corrupt. This can occur if - ** one cursor has modified page pParent while a reference to it is held - ** by a second cursor. Which can only happen if a single page is linked - ** into more than one b-tree structure in a corrupt database. */ -#if 0 assertParentIndex( pCur->apPage[pCur->iPage-1], pCur->aiIdx[pCur->iPage-1], pCur->apPage[pCur->iPage]->pgno ); -#endif testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); - - releasePage(pCur->apPage[pCur->iPage]); - pCur->iPage--; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + releasePageNotNull(pCur->apPage[pCur->iPage--]); } /* @@ -56666,18 +59316,23 @@ static int moveToRoot(BtCursor *pCur){ } if( pCur->iPage>=0 ){ - while( pCur->iPage ) releasePage(pCur->apPage[pCur->iPage--]); + while( pCur->iPage ){ + assert( pCur->apPage[pCur->iPage]!=0 ); + releasePageNotNull(pCur->apPage[pCur->iPage--]); + } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ + assert( pCur->iPage==(-1) ); rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; + pCur->curIntKey = pCur->apPage[0]->intKey; } pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); @@ -56880,7 +59535,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->apPage[0]->intKey + && pCur->curIntKey ){ if( pCur->info.nKey==intKey ){ *pRes = 0; @@ -56915,7 +59570,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } - assert( pCur->apPage[0]->intKey || pIdxKey ); + assert( pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey || pIdxKey ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; @@ -56938,7 +59594,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; - pCell = findCell(pPage, idx) + pPage->childPtrSize; + pCell = findCellPastPtr(pPage, idx); if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; @@ -56970,8 +59626,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } }else{ for(;;){ - int nCell; - pCell = findCell(pPage, idx) + pPage->childPtrSize; + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree @@ -56999,12 +59655,25 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. */ + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; - btreeParseCellPtr(pPage, pCellBody, &pCur->info); + pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; - pCellKey = sqlite3Malloc( nCell ); + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto moveto_finish; + } + pCellKey = sqlite3Malloc( nCell+18 ); if( pCellKey==0 ){ rc = SQLITE_NOMEM; goto moveto_finish; @@ -57297,8 +59966,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** sqlite3PagerUnref() on the new page when it is done. ** ** SQLITE_OK is returned on success. Any other return value indicates -** an error. *ppPage and *pPgno are undefined in the event of an error. -** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** an error. *ppPage is set to NULL in the event of an error. ** ** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an @@ -57330,6 +59998,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -57339,6 +60009,7 @@ static int allocateBtreePage( /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + u32 nSearch = 0; /* Count of the number of search attempts */ /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then @@ -57376,15 +60047,21 @@ static int allocateBtreePage( do { pPrevTrunk = pTrunk; if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); - if( iTrunk>mxPage ){ + if( iTrunk>mxPage || nSearch++ > n ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); } if( rc ){ pTrunk = 0; @@ -57392,8 +60069,9 @@ static int allocateBtreePage( } assert( pTrunk!=0 ); assert( pTrunk->aData!=0 ); - - k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ /* The trunk has no leaves and the list is not being searched. ** So extract the trunk page itself and use it as the newly @@ -57448,7 +60126,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -57528,11 +60206,12 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } } searchList = 0; @@ -57576,7 +60255,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -57590,11 +60269,12 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } @@ -57604,17 +60284,8 @@ static int allocateBtreePage( end_allocate_page: releasePage(pTrunk); releasePage(pPrevTrunk); - if( rc==SQLITE_OK ){ - if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ - releasePage(*ppPage); - *ppPage = 0; - return SQLITE_CORRUPT_BKPT; - } - (*ppPage)->isInit = 0; - }else{ - *ppPage = 0; - } - assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); + assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); + assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); return rc; } @@ -57639,9 +60310,10 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ int nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( iPage>1 ); + assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); + if( iPage<2 ) return SQLITE_CORRUPT_BKPT; if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -57711,6 +60383,11 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** for now. At some point in the future (once everyone has upgraded ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. */ rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc==SQLITE_OK ){ @@ -57776,7 +60453,7 @@ static int clearCell( u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); *pnSize = info.nSize; if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ @@ -57788,7 +60465,9 @@ static int clearCell( assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( ovflPgno==0 || nOvfl>0 ); + assert( nOvfl>0 || + (CORRUPT_DB && (info.nPayload + ovflPageSize)0x7fffffff || pKey==0) ){ - return SQLITE_CORRUPT_BKPT; - } + assert( nKey<=0x7fffffff && pKey!=0 ); nPayload = (int)nKey; pSrc = pKey; nSrc = (int)nKey; @@ -57928,7 +60605,7 @@ static int fillInCell( #if SQLITE_DEBUG { CellInfo info; - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); assert( nHeader=(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); assert( *pnSize == info.nSize ); @@ -58043,7 +60720,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ if( *pRC ) return; assert( idx>=0 && idxnCell ); - assert( sz==cellSize(pPage, idx) ); + assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; @@ -58062,9 +60739,17 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } pPage->nCell--; - memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } } /* @@ -58090,10 +60775,8 @@ static void insertCell( ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ - int end; /* First byte past the last cell pointer in data[] */ - int ins; /* Index in data[] where new cell pointer is inserted */ - int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ if( *pRC ) return; @@ -58108,7 +60791,7 @@ static void insertCell( ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size ** might be less than 8 (leaf-size + pointer) on the interior node. Hence ** the term after the || in the following assert(). */ - assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); @@ -58121,6 +60804,14 @@ static void insertCell( assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ @@ -58129,24 +60820,26 @@ static void insertCell( } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; - cellOffset = pPage->cellOffset; - end = cellOffset + 2*pPage->nCell; - ins = cellOffset + 2*i; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); if( rc ){ *pRC = rc; return; } - /* The allocateSpace() routine guarantees the following two properties - ** if it returns success */ - assert( idx >= end+2 ); + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); - pPage->nCell++; pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } - memmove(&data[ins+2], &data[ins], end-ins); - put2byte(&data[ins], idx); - put2byte(&data[pPage->hdrOffset+3], pPage->nCell); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -58159,45 +60852,328 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** A CellArray object contains a cache of pointers and sizes for a +** consecutive sequence of cells that might be held multiple pages. +*/ +typedef struct CellArray CellArray; +struct CellArray { + int nCell; /* Number of cells in apCell[] */ + MemPage *pRef; /* Reference page */ + u8 **apCell; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ +}; + +/* +** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been +** computed. +*/ +static void populateCellCache(CellArray *p, int idx, int N){ + assert( idx>=0 && idx+N<=p->nCell ); + while( N>0 ){ + assert( p->apCell[idx]!=0 ); + if( p->szCell[idx]==0 ){ + p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + }else{ + assert( CORRUPT_DB || + p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + } + idx++; + N--; + } +} + +/* +** Return the size of the Nth element of the cell array */ -static void assemblePage( - MemPage *pPage, /* The page to be assembled */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ +static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + assert( p->szCell[N]==0 ); + p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); + return p->szCell[N]; +} +static u16 cachedCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + if( p->szCell[N] ) return p->szCell[N]; + return computeCellSize(p, N); +} + +/* +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. +*/ +static int rebuildPage( + MemPage *pPg, /* Edit this page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); + pData = pEnd; + for(i=0; iaData && pCellxCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + } + + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; + return SQLITE_OK; +} - pCellptr = &pPage->aCellIdx[nCell*2]; - cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ - u16 sz = aSize[i]; - pCellptr -= 2; - cellbody -= sz; - put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], sz); +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ +static int pageInsertArray( + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int iFirst, /* Index of first cell to add */ + int nCell, /* Number of cells to add to pPg */ + CellArray *pCArray /* Array of cells */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + int iEnd = iFirst + nCell; + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ + for(i=iFirst; iapCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + memmove(pSlot, pCArray->apCell[i], sz); + put2byte(pCellptr, (pSlot - aData)); + pCellptr += 2; + } + *ppData = pData; + return 0; +} + +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function adds the +** space associated with each cell in the array that is currently stored +** within the body of pPg to the pPg free-list. The cell-pointers and other +** fields of the page are not updated. +** +** This function returns the total number of cells added to the free-list. +*/ +static int pageFreeArray( + MemPage *pPg, /* Page to edit */ + int iFirst, /* First cell to delete */ + int nCell, /* Cells to delete */ + CellArray *pCArray /* Array of cells */ +){ + u8 * const aData = pPg->aData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; + int nRet = 0; + int i; + int iEnd = iFirst + nCell; + u8 *pFree = 0; + int szFree = 0; + + for(i=iFirst; iapCell[i]; + if( pCell>=pStart && pCellszCell[i]; assert( sz>0 ); + if( pFree!=(pCell + sz) ){ + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + pFree = pCell; + szFree = sz; + if( pFree+sz>pEnd ) return 0; + }else{ + pFree = pCell; + szFree += sz; + } + nRet++; + } } - put2byte(&data[hdr+3], nCell); - put2byte(&data[hdr+5], cellbody); - pPage->nFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + return nRet; +} + +/* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** +** The pPg->nFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ +static int editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + CellArray *pCArray /* Array of cells and sizes */ +){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + } + + pData = &aData[get2byteNotZero(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew, nAdd, pCArray + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iCell+iNew, 1, pCArray + ) ) goto editpage_fail; + } + } + + /* Append cells to the end of the page */ + pCellptr = &pPg->aCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew+nCell, nNew-nCell, pCArray + ) ) goto editpage_fail; + + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iapCell[i+iNew]; + int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], + pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); + } +#endif + + return SQLITE_OK; + editpage_fail: + /* Unable to edit this page. Rebuild it from scratch instead. */ + populateCellCache(pCArray, iNew, nNew); + return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); } /* @@ -58251,7 +61227,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -58263,13 +61239,15 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pOut = &pSpace[4]; u8 *pCell = pPage->apOvfl[0]; - u16 szCell = cellSizePtr(pPage, pCell); + u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rc = rebuildPage(pNew, 1, &pCell, &szCell); + if( NEVER(rc) ) return rc; + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -58341,7 +61319,7 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ u8 *z; z = findCell(pPage, j); - btreeParseCellPtr(pPage, z, &info); + pPage->xParseCell(pPage, z, &info); if( info.iOverflow ){ Pgno ovfl = get4byte(&z[info.iOverflow]); ptrmapGet(pBt, ovfl, &e, &n); @@ -58472,7 +61450,6 @@ static int balance_nonroot( int bBulk /* True if this call is part of a bulk load */ ){ BtShared *pBt; /* The whole database */ - int nCell = 0; /* Number of cells in apCell[] */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nNew = 0; /* Number of pages in apNew[] */ int nOld; /* Number of pages in apOld[] */ @@ -58483,22 +61460,27 @@ static int balance_nonroot( int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ int usableSpace; /* Bytes in pPage beyond the header */ int pageFlags; /* Value of pPage->aData[0] */ - int subtotal; /* Subtotal of bytes in cells on one page */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ - int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ - u8 **apCell = 0; /* All cells begin balanced */ - u16 *szCell; /* Local size of all cells in apCell[] */ + int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in b.apCell[] */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + CellArray b; /* Parsed information on cells being balanced */ + + memset(abDone, 0, sizeof(abDone)); + b.nCell = 0; + b.apCell = 0; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -58540,7 +61522,6 @@ static int balance_nonroot( }else if( iParentIdx==i ){ nxDiv = i-2+bBulk; }else{ - assert( bBulk==0 ); nxDiv = iParentIdx-1; } i = 2-bBulk; @@ -58553,7 +61534,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -58564,12 +61545,12 @@ static int balance_nonroot( if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); pParent->nOverflow = 0; }else{ apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); /* Drop the cell from the parent page. apDiv[i] still points to ** the cell within the parent, even though it has been dropped. @@ -58607,138 +61588,209 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = - nMaxCells*sizeof(u8*) /* apCell */ - + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ - apCell = sqlite3ScratchMalloc( szScratch ); - if( apCell==0 ){ + nMaxCells*sizeof(u8*) /* b.apCell */ + + nMaxCells*sizeof(u16) /* b.szCell */ + + pBt->pageSize; /* aSpace1 */ + + /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size. */ + assert( szScratch<=6*(int)pBt->pageSize ); + b.apCell = sqlite3ScratchMalloc( szScratch ); + if( b.apCell==0 ){ rc = SQLITE_NOMEM; goto balance_cleanup; } - szCell = (u16*)&apCell[nMaxCells]; - aSpace1 = (u8*)&szCell[nMaxCells]; + b.szCell = (u16*)&b.apCell[nMaxCells]; + aSpace1 = (u8*)&b.szCell[nMaxCells]; assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); /* ** Load pointers to all cells on sibling pages and the divider cells - ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into the local b.apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied - ** into aSpace1[]. In this way, all cells in apCell[] are without + ** into aSpace1[]. In this way, all cells in b.apCell[] are without ** child pointers. If siblings are not leaves, then all cell in - ** apCell[] include child pointers. Either way, all cells in apCell[] + ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] ** are alike. ** ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ - leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->intKeyLeaf; + b.pRef = apOld[0]; + leafCorrection = b.pRef->leaf*4; + leafData = b.pRef->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); - - limit = pOld->nCell+pOld->nOverflow; + MemPage *pOld = apOld[i]; + int limit = pOld->nCell; + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; + + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + + /* Load b.apCell[] with pointers to all cells in pOld. If pOld + ** constains overflow cells, include them in the b.apCell[] array + ** in the correct spot. + ** + ** Note that when there are multiple overflow cells, it is always the + ** case that they are sequential and adjacent. This invariant arises + ** because multiple overflows can only occurs when inserting divider + ** cells into a parent on a prior balance, and divider cells are always + ** adjacent and are inserted in order. There is an assert() tagged + ** with "NOTE 1" in the overflow cell insertion loop to prove this + ** invariant. + ** + ** This must be done in advance. Once the balance starts, the cell + ** offset section of the btree page will be overwritten and we will no + ** long be able to find the cells if a pointer to each cell is not saved + ** first. + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit); if( pOld->nOverflow>0 ){ + memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow); + limit = pOld->aiOvfl[0]; for(j=0; jaData; - u16 maskPage = pOld->maskPage; - u16 cellOffset = pOld->cellOffset; - for(j=0; jnOverflow; k++){ + assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ + b.apCell[b.nCell] = pOld->apOvfl[k]; + b.nCell++; } - } + } + piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + while( piCellmaxLocal+23 ); assert( iSpace1 <= (int)pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); - apCell[nCell] = pTemp+leafCorrection; + b.apCell[b.nCell] = pTemp+leafCorrection; assert( leafCorrection==0 || leafCorrection==4 ); - szCell[nCell] = szCell[nCell] - leafCorrection; + b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; if( !pOld->leaf ){ assert( leafCorrection==0 ); assert( pOld->hdrOffset==0 ); /* The right pointer of the child page pOld becomes the left ** pointer of the divider cell */ - memcpy(apCell[nCell], &pOld->aData[8], 4); + memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); }else{ assert( leafCorrection==4 ); - if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ - szCell[nCell] = 4; + while( b.szCell[b.nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); + assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); + aSpace1[iSpace1++] = 0x00; + b.szCell[b.nCell]++; } } - nCell++; + b.nCell++; } } /* - ** Figure out the number of pages needed to hold all nCell cells. + ** Figure out the number of pages needed to hold all b.nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in apCell[] of the cell that divides page i from page i+1. - ** cntNew[k] should equal nCell. + ** in b.apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal b.nCell. ** ** Values computed by this block: ** ** k: The total number of sibling pages ** szNew[i]: Spaced used on the i-th sibling page. - ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to + ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(subtotal=k=i=0; i usableSpace ){ - szNew[k] = subtotal - szCell[i]; - cntNew[k] = i; - if( leafData ){ i--; } - subtotal = 0; - k++; - if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - } - } - szNew[k] = subtotal; - cntNew[k] = nCell; - k++; + for(i=0; inFree; + if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + for(j=0; jnOverflow; j++){ + szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); + } + cntNew[i] = cntOld[i]; + } + k = nOld; + for(i=0; iusableSpace ){ + if( i+1>=k ){ + k = i+2; + if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + szNew[k-1] = 0; + cntNew[k-1] = b.nCell; + } + sz = 2 + cachedCellSize(&b, cntNew[i]-1); + szNew[i] -= sz; + if( !leafData ){ + if( cntNew[i]usableSpace ) break; + szNew[i] += sz; + cntNew[i]++; + if( !leafData ){ + if( cntNew[i]=b.nCell ){ + k = i+1; + }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -58752,46 +61804,46 @@ static int balance_nonroot( r = cntNew[i-1] - 1; d = r + 1 - leafData; - assert( d szLeft-(b.szCell[r]+2)) ){ + break; + } + szRight += b.szCell[d] + 2; + szLeft -= b.szCell[r] + 2; + cntNew[i-1] = r; + r--; + d--; + }while( r>=0 ); szNew[i] = szRight; szNew[i-1] = szLeft; + if( cntNew[i-1] <= (i>1 ? cntNew[i-2] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* ** Allocate k new pages. Reuse old pages where possible. */ - if( apOld[0]->pgno<=1 ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } pageFlags = apOld[0]->aData[0]; for(i=0; i0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; + cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ if( ISAUTOVACUUM ){ @@ -58818,135 +61872,250 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; ji ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + } + for(i=0; ii ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. + /* If the sibling pages are not leaves, ensure that the right-child pointer + ** of the right-most new sibling page is set to the value that was + ** originally in the same field of the right-most old sibling page. */ + if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ + MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } + + /* Make any required updates to pointer map entries associated with + ** cells stored on sibling pages following the balance operation. Pointer + ** map entries associated with divider cells are set by the insertCell() + ** routine. The associated pointer map entries are: + ** + ** a) if the cell contains a reference to an overflow chain, the + ** entry associated with the first page in the overflow chain, and + ** + ** b) if the sibling pages are not leaves, the child page associated + ** with the cell. + ** + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been + ** populated, not here. */ - j = 0; - for(i=0; iaData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; + + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow cell), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || pCell=&aOld[usableSize] + ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( cachedCellSize(&b,i)>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); + } + if( rc ) goto balance_cleanup; + } + } + } + + /* Insert new divider cells into pParent. */ + for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + assert( b.apCell[j]!=0 ); + pCell = b.apCell[j]; + sz = b.szCell[j] + leafCorrection; + pTemp = &aOvflSpace[iOvflSpace]; + if( !pNew->leaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + pNew->xParseCell(pNew, b.apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( b.szCell[j]==4 ){ + assert(leafCorrection==4); + sz = pParent->xCellSize(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } - j = cntNew[i]; + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. + ** + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. + ** + ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. + */ + for(i=1-nNew; i=0 && iPg=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ + ){ + int iNew; + int iOld; + int nNewCell; - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( i=cntNew[iPg-1] || abDone[iPg-1] ); - assert( jleaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); - } + iOld = iPgmaxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - j++; - nxDiv++; + rc = editPage(apNew[iPg], iOld, iNew, nNewCell, &b); + if( rc ) goto balance_cleanup; + abDone[iPg]++; + apNew[iPg]->nFree = usableSpace-szNew[iPg]; + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); } } - assert( j==nCell ); + + /* All pages have been processed exactly once */ + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -58959,132 +62128,56 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ - assert( nNew==1 ); + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ + assert( nNew==1 || CORRUPT_DB ); + rc = defragmentPage(apNew[0]); + testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, b.nCell)); + + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(apCell); + sqlite3ScratchFree(b.apCell); for(i=0; ipBt->pageSize); - rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, + pCur->hints&BTREE_BULKLOAD); if( pFree ){ /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are @@ -59278,6 +62372,7 @@ static int balance(BtCursor *pCur){ /* The next iteration of the do-loop balances the parent page. */ releasePage(pPage); pCur->iPage--; + assert( pCur->iPage>=0 ); } }while( rc==SQLITE_OK ); @@ -59357,24 +62452,28 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** doing any work. To avoid thwarting these optimizations, it is important ** not to clear the cursor here. */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } if( pCur->pKeyInfo==0 ){ + assert( pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ invalidateIncrblobCursors(p, nKey, 0); /* If the cursor is currently on the last row and we are appending a - ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() - ** call */ + ** new row onto the end, set the "loc" to avoid an unnecessary + ** btreeMoveto() call */ if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ - loc = -1; + loc = -1; + }else if( loc==0 ){ + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc); + if( rc ) return rc; } - } - - if( !loc ){ + }else if( loc==0 ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; } @@ -59392,7 +62491,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( newCell!=0 ); rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; - assert( szNew==cellSizePtr(pPage, newCell) ); + assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->aiIdx[pCur->iPage]; if( loc==0 ){ @@ -59457,10 +62556,15 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at an arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the second parameter is zero, then the cursor is left pointing at an +** arbitrary location after the delete. If it is non-zero, then the cursor +** is left in a state such that the next call to BtreeNext() or BtreePrev() +** moves it to the same row as it would if the call to BtreeDelete() had +** been omitted. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, int bPreserve){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -59469,6 +62573,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -59476,12 +62581,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - - if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) - || NEVER(pCur->eState!=CURSOR_VALID) - ){ - return SQLITE_ERROR; /* Something has gone awry. */ - } + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; @@ -59502,12 +62603,11 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; + ** making any modifications. */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ @@ -59515,6 +62615,31 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -59533,7 +62658,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pTmp; pCell = findCell(pLeaf, pLeaf->nCell-1); - nCell = cellSizePtr(pLeaf, pCell); + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); @@ -59567,7 +62693,23 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && pCur->iPage==iCellDepth ); + assert( pPage==pCur->apPage[pCur->iPage] ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } @@ -59625,7 +62767,8 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } - assert( pgnoRoot>=3 ); + assert( pgnoRoot>=3 || CORRUPT_DB ); + testcase( pgnoRoot<3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens @@ -59754,9 +62897,13 @@ static int clearDatabasePage( if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - - rc = getAndInitPage(pBt, pgno, &pPage, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); if( rc ) return rc; + if( pPage->bBusy ){ + rc = SQLITE_CORRUPT_BKPT; + goto cleardatabasepage_out; + } + pPage->bBusy = 1; hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -59771,7 +62918,8 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ - assert( pPage->intKey ); + assert( pPage->intKey || CORRUPT_DB ); + testcase( !pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ @@ -59781,6 +62929,7 @@ static int clearDatabasePage( } cleardatabasepage_out: + pPage->bBusy = 0; releasePage(pPage); return rc; } @@ -59970,6 +63119,13 @@ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. */ SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ BtShared *pBt = p->pBt; @@ -59980,7 +63136,11 @@ SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ assert( pBt->pPage1 ); assert( idx>=0 && idx<=15 ); - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion; + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ @@ -60071,7 +63231,7 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ if( pCur->iPage==0 ){ /* All pages of the b-tree have been visited. Return successfully. */ *pnEntry = nEntry; - return SQLITE_OK; + return moveToRoot(pCur); } moveToParent(pCur); }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); @@ -60114,7 +63274,6 @@ static void checkAppendMsg( ... ){ va_list ap; - char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -60123,8 +63282,7 @@ static void checkAppendMsg( sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); - sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); + sqlite3XPrintf(&pCheck->errMsg, 0, pCheck->zPfx, pCheck->v1, pCheck->v2); } sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); @@ -60273,10 +63431,65 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); + + if( isFreeList && N<(iPage!=0) ){ + checkAppendMsg(pCheck, "free-page count in header is too small"); + } } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ +/* +** An implementation of a min-heap. +** +** aHeap[0] is the number of elements on the heap. aHeap[1] is the +** root element. The daughter nodes of aHeap[N] are aHeap[N*2] +** and aHeap[N*2+1]. +** +** The heap property is this: Every node is less than or equal to both +** of its daughter nodes. A consequence of the heap property is that the +** root node aHeap[1] is always the minimum value currently in the heap. +** +** The btreeHeapInsert() routine inserts an unsigned 32-bit number onto +** the heap, preserving the heap property. The btreeHeapPull() routine +** removes the root element from the heap (the minimum value in the heap) +** and then moves other nodes around as necessary to preserve the heap +** property. +** +** This heap is used for cell overlap and coverage testing. Each u32 +** entry represents the span of a cell or freeblock on a btree page. +** The upper 16 bits are the index of the first byte of a range and the +** lower 16 bits are the index of the last byte of that range. +*/ +static void btreeHeapInsert(u32 *aHeap, u32 x){ + u32 j, i = ++aHeap[0]; + aHeap[i] = x; + while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ + x = aHeap[j]; + aHeap[j] = aHeap[i]; + aHeap[i] = x; + i = j; + } +} +static int btreeHeapPull(u32 *aHeap, u32 *pOut){ + u32 j, i, x; + if( (x = aHeap[0])==0 ) return 0; + *pOut = aHeap[1]; + aHeap[1] = aHeap[x]; + aHeap[x] = 0xffffffff; + aHeap[0]--; + i = 1; + while( (j = i*2)<=aHeap[0] ){ + if( aHeap[j]>aHeap[j+1] ) j++; + if( aHeap[i]zPfx; int saved_v1 = pCheck->v1; int saved_v2 = pCheck->v2; + u8 savedIsInit = 0; /* Check that the page exists */ @@ -60327,54 +63548,95 @@ static int checkTreePage( if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - depth = -1; goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); - releasePage(pPage); - depth = -1; goto end_of_check; } + data = pPage->aData; + hdr = pPage->hdrOffset; - /* Check out all the cells. - */ - depth = 0; - for(i=0; inCell && pCheck->mxErr; i++){ - u8 *pCell; - u32 sz; + /* Set up for cell analysis */ + pCheck->zPfx = "On tree page %d cell %d: "; + contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + assert( pPage->nCell==nCell ); + + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + assert( pPage->aCellIdx==&data[cellStart] ); + pCellIdx = &data[cellStart + 2*(nCell-1)]; + + if( !pPage->leaf ){ + /* Analyze the right-child page of internal pages */ + pgno = get4byte(&data[hdr+8]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + pCheck->zPfx = "On page %d at right child: "; + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + }else{ + /* For leaf pages, the coverage check will occur in the same loop + ** as the other cell checks, so initialize the heap. */ + heap = pCheck->heap; + heap[0] = 0; + } + + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ CellInfo info; - /* Check payload overflow pages - */ - pCheck->zPfx = "On tree page %d cell %d: "; - pCheck->v1 = iPage; + /* Check cell size */ pCheck->v2 = i; - pCell = findCell(pPage,i); - btreeParseCellPtr(pPage, pCell, &info); - sz = info.nPayload; - /* For intKey pages, check that the keys are in order. - */ + assert( pCellIdx==&data[cellStart + i*2] ); + pc = get2byteAligned(pCellIdx); + pCellIdx -= 2; + if( pcusableSize-4 ){ + checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + pc, contentOffset, usableSize-4); + doCoverageCheck = 0; + continue; + } + pCell = &data[pc]; + pPage->xParseCell(pPage, pCell, &info); + if( pc+info.nSize>usableSize ){ + checkAppendMsg(pCheck, "Extends off end of page"); + doCoverageCheck = 0; + continue; + } + + /* Check for integer primary key out of range */ if( pPage->intKey ){ - if( i==0 ){ - nMinKey = nMaxKey = info.nKey; - }else if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ + checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); } - nMaxKey = info.nKey; + maxKey = info.nKey; } - if( (sz>info.nLocal) - && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) - ){ - int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); - Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); + + /* Check the content overflow list */ + if( info.nPayload>info.nLocal ){ + int nPage; /* Number of pages on the overflow chain */ + Pgno pgnoOvfl; /* First page of the overflow chain */ + assert( pc + info.iOverflow <= usableSize ); + nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); + pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); @@ -60383,134 +63645,109 @@ static int checkTreePage( checkList(pCheck, 0, pgnoOvfl, nPage); } - /* Check sanity of left child page. - */ if( !pPage->leaf ){ + /* Check sanity of left child page for internal pages */ pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); - if( i>0 && d2!=depth ){ + d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + if( d2!=depth ){ checkAppendMsg(pCheck, "Child page depth differs"); + depth = d2; } - depth = d2; - } - } - - if( !pPage->leaf ){ - pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - pCheck->zPfx = "On page %d at right child: "; - pCheck->v1 = iPage; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); - } -#endif - checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); - } - - /* For intKey leaf pages, check that the min/max keys are in order - ** with any left/parent/right pages. - */ - pCheck->zPfx = "Page %d: "; - pCheck->v1 = iPage; - if( pPage->leaf && pPage->intKey ){ - /* if we are a left child page */ - if( pnParentMinKey ){ - /* if we are the left most child page */ - if( !pnParentMaxKey ){ - if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (max larger than parent min of %lld)", - nMaxKey, *pnParentMinKey); - } - }else{ - if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (min less than parent min of %lld)", - nMinKey, *pnParentMinKey); - } - if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (max larger than parent max of %lld)", - nMaxKey, *pnParentMaxKey); - } - *pnParentMinKey = nMaxKey; - } - /* else if we're a right child page */ - } else if( pnParentMaxKey ){ - if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (min less than parent max of %lld)", - nMinKey, *pnParentMaxKey); - } + }else{ + /* Populate the coverage-checking heap for leaf pages */ + btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); } } + *piMinKey = maxKey; /* Check for complete coverage of the page */ - data = pPage->aData; - hdr = pPage->hdrOffset; - hit = sqlite3PageMalloc( pBt->pageSize ); pCheck->zPfx = 0; - if( hit==0 ){ - pCheck->mallocFailed = 1; - }else{ - int contentOffset = get2byteNotZero(&data[hdr+5]); - assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ - memset(hit+contentOffset, 0, usableSize-contentOffset); - memset(hit, 1, contentOffset); - nCell = get2byte(&data[hdr+3]); - cellStart = hdr + 12 - 4*pPage->leaf; - for(i=0; i=usableSize ){ - pCheck->zPfx = 0; - checkAppendMsg(pCheck, - "Corruption detected in cell %d on page %d",i,iPage); - }else{ - for(j=pc+size-1; j>=pc; j--) hit[j]++; + if( doCoverageCheck && pCheck->mxErr>0 ){ + /* For leaf pages, the min-heap has already been initialized and the + ** cells have already been inserted. But for internal pages, that has + ** not yet been done, so do it now */ + if( !pPage->leaf ){ + heap = pCheck->heap; + heap[0] = 0; + for(i=nCell-1; i>=0; i--){ + u32 size; + pc = get2byteAligned(&data[cellStart+i*2]); + size = pPage->xCellSize(pPage, &data[pc]); + btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + /* Add the freeblocks to the min-heap + ** + ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. + */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ size = get2byte(&data[i+2]); - assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ - for(j=i+size-1; j>=i; j--) hit[j]++; + assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; } - for(i=cnt=0; i1 ){ + /* Analyze the min-heap looking for overlap between cells and/or + ** freeblocks, and counting the number of untracked bytes in nFrag. + ** + ** Each min-heap entry is of the form: (start_address<<16)|end_address. + ** There is an implied first entry the covers the page header, the cell + ** pointer index, and the gap between the cell pointer index and the start + ** of cell content. + ** + ** The loop below pulls entries from the min-heap in order and compares + ** the start_address against the previous end_address. If there is an + ** overlap, that means bytes are used multiple times. If there is a gap, + ** that gap is added to the fragmentation count. + */ + nFrag = 0; + prev = contentOffset - 1; /* Implied first min-heap entry */ + while( btreeHeapPull(heap,&x) ){ + if( (prev&0xffff)>=(x>>16) ){ checkAppendMsg(pCheck, - "Multiple uses for byte %d of page %d", i, iPage); + "Multiple uses for byte %u of page %d", x>>16, iPage); break; + }else{ + nFrag += (x>>16) - (prev&0xffff) - 1; + prev = x; } } - if( cnt!=data[hdr+7] ){ + nFrag += usableSize - (prev&0xffff) - 1; + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ + if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", - cnt, data[hdr+7], iPage); + nFrag, data[hdr+7], iPage); } } - sqlite3PageFree(hit); - releasePage(pPage); end_of_check: + if( !doCoverageCheck ) pPage->isInit = savedIsInit; + releasePage(pPage); pCheck->zPfx = saved_zPfx; pCheck->v1 = saved_v1; pCheck->v2 = saved_v2; @@ -60540,14 +63777,15 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( int *pnErr /* Write number of errors seen to this variable */ ){ Pgno i; - int nRef; IntegrityCk sCheck; BtShared *pBt = p->pBt; + int savedDbFlags = pBt->db->flags; char zErr[100]; + VVA_ONLY( int nRef ); sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - nRef = sqlite3PagerRefcount(pBt->pPager); + assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 ); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); @@ -60557,22 +63795,26 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.zPfx = 0; sCheck.v1 = 0; sCheck.v2 = 0; - *pnErr = 0; + sCheck.aPgRef = 0; + sCheck.heap = 0; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); if( sCheck.nPage==0 ){ - sqlite3BtreeLeave(p); - return 0; + goto integrity_ck_cleanup; } sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ - *pnErr = 1; - sqlite3BtreeLeave(p); - return 0; + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; } + sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + if( sCheck.heap==0 ){ + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; + } + i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); - sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist */ @@ -60583,17 +63825,19 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - sCheck.zPfx = "List of tree roots: "; - checkTreePage(&sCheck, aRoot[i], NULL, NULL); - sCheck.zPfx = 0; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); } + pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ @@ -60617,28 +63861,20 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( #endif } - /* Make sure this analysis did not leave any unref() pages. - ** This is an internal consistency check; an integrity check - ** of the integrity check. - */ - if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, - "Outstanding page count goes from %d to %d during this analysis", - nRef, sqlite3PagerRefcount(pBt->pPager) - ); - } - /* Clean up and report errors. */ - sqlite3BtreeLeave(p); +integrity_ck_cleanup: + sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); - *pnErr = sCheck.nErr+1; - return 0; + sCheck.nErr++; } *pnErr = sCheck.nErr; if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + /* Make sure this analysis did not leave any unref() pages. */ + assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); + sqlite3BtreeLeave(p); return sqlite3StrAccumFinish(&sCheck.errMsg); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -60849,6 +64085,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void */ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ pCur->curFlags |= BTCF_Incrblob; + pCur->pBtree->hasIncrblobCur = 1; } #endif @@ -60889,14 +64126,23 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ } /* -** set the mask of hint flags for cursor pCsr. Currently the only valid -** values are 0 and BTREE_BULKLOAD. +** set the mask of hint flags for cursor pCsr. */ SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ - assert( mask==BTREE_BULKLOAD || mask==0 ); + assert( mask==BTREE_BULKLOAD || mask==BTREE_SEEK_EQ || mask==0 ); pCsr->hints = mask; } +#ifdef SQLITE_DEBUG +/* +** Return true if the cursor has a hint specified. This routine is +** only used from within assert() statements +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ + return (pCsr->hints & mask)!=0; +} +#endif + /* ** Return true if the given Btree is read-only. */ @@ -60904,6 +64150,11 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; } +/* +** Return the size of the header added to each page by this module. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + /************** End of btree.c ***********************************************/ /************** Begin file backup.c ******************************************/ /* @@ -60920,6 +64171,8 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ ** This file contains the implementation of the sqlite3_backup_XXX() ** API functions and the related features. */ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ /* ** Structure allocated for each backup operation. @@ -61028,6 +64281,20 @@ static int setDestPgsz(sqlite3_backup *p){ return rc; } +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeIsInReadTrans(p) ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + /* ** Create an sqlite3_backup process to copy the contents of zSrcDb from ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return @@ -61036,7 +64303,7 @@ static int setDestPgsz(sqlite3_backup *p){ ** If an error occurs, NULL is returned and an error code and error message ** stored in database handle pDestDb. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( +SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( sqlite3* pDestDb, /* Database to write to */ const char *zDestDb, /* Name of database within pDestDb */ sqlite3* pSrcDb, /* Database connection to read from */ @@ -61044,6 +64311,13 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -61080,12 +64354,15 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + if( 0==p->pSrc || 0==p->pDest + || setDestPgsz(p)==SQLITE_NOMEM + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ /* One (or both) of the named databases did not exist or an OOM - ** error was hit. The error has already been written into the - ** pDestDb handle. All that is left to do here is free the - ** sqlite3_backup structure. - */ + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ sqlite3_free(p); p = 0; } @@ -61129,7 +64406,7 @@ static int backupOnePage( ** guaranteed that the shared-mutex is held by this thread, handle ** p->pSrc may not actually be the owner. */ int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); - int nDestReserve = sqlite3BtreeGetReserve(p->pDest); + int nDestReserve = sqlite3BtreeGetOptimalReserve(p->pDest); #endif int rc = SQLITE_OK; i64 iOff; @@ -61234,12 +64511,15 @@ static void attachBackupObject(sqlite3_backup *p){ /* ** Copy nPage pages from the source b-tree to the destination. */ -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ int rc; int destMode; /* Destination journal mode */ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -61476,7 +64756,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* ** Release all resources associated with an sqlite3_backup* handle. */ -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p){ sqlite3_backup **pp; /* Ptr to head of pagers backup list */ sqlite3 *pSrcDb; /* Source database connection */ int rc; /* Value to return */ @@ -61528,7 +64808,13 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ ** Return the number of pages still to be backed up as of the most recent ** call to sqlite3_backup_step(). */ -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -61536,7 +64822,13 @@ SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ ** Return the total number of pages in the source database as of the most ** recent call to sqlite3_backup_step(). */ -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } @@ -61552,9 +64844,13 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ ** corresponding to the source database is held when this function is ** called. */ -SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ - sqlite3_backup *p; /* Iterator variable */ - for(p=pBackup; p; p=p->pNext){ +static SQLITE_NOINLINE void backupUpdate( + sqlite3_backup *p, + Pgno iPage, + const u8 *aData +){ + assert( p!=0 ); + do{ assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); if( !isFatalError(p->rc) && iPageiNext ){ /* The backup process p has already copied page iPage. But now it @@ -61571,7 +64867,10 @@ SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, con p->rc = rc; } } - } + }while( (p = p->pNext)!=0 ); +} +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + if( pBackup ) backupUpdate(pBackup, iPage, aData); } /* @@ -61629,6 +64928,10 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; +#ifdef SQLITE_HAS_CODEC + sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +#endif + /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes @@ -61672,6 +64975,8 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ ** only within the VDBE. Interface routines refer to a Mem using the ** name sqlite_value */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifdef SQLITE_DEBUG /* @@ -61855,10 +65160,11 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; pMem->flags |= MEM_Term; + } + pMem->flags &= ~MEM_Ephem; #ifdef SQLITE_DEBUG - pMem->pScopyFrom = 0; + pMem->pScopyFrom = 0; #endif - } return SQLITE_OK; } @@ -62242,7 +65548,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ if( pMem->flags & MEM_Null ) return; switch( aff ){ - case SQLITE_AFF_NONE: { /* Really a cast to BLOB */ + case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ if( (pMem->flags & MEM_Blob)==0 ){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); @@ -62424,10 +65730,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ -/* -** Size of struct Mem not including the Mem.zMalloc member. -*/ -#define MEMCELLSIZE offsetof(Mem,zMalloc) /* ** Make an shallow copy of pFrom into pTo. Prior contents of @@ -62435,10 +65737,15 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ ** pFrom->z is used, then pTo->z points to the same thing as pFrom->z ** and flags gets srcType (either MEM_Ephem or MEM_Static). */ +static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ + vdbeMemClearExternAndSetNull(pTo); + assert( !VdbeMemDynamic(pTo) ); + sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); +} SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); assert( pTo->db==pFrom->db ); - if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); + if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); @@ -62454,7 +65761,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; - assert( pTo->db==pFrom->db ); + /* The pFrom==0 case in the following assert() is when an sqlite3_value + ** from sqlite3_value_dup() is used as the argument + ** to sqlite3_result_value(). */ + assert( pTo->db==pFrom->db || pFrom->db==0 ); assert( (pFrom->flags & MEM_RowSet)==0 ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); @@ -62601,6 +65911,32 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ +static SQLITE_NOINLINE int vdbeMemFromBtreeResize( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + int key, /* If true, retrieve from the btree key, not data. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + int rc; + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } + } + return rc; +} SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ @@ -62630,22 +65966,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; }else{ - pMem->flags = MEM_Null; - if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } - if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; - pMem->n = (int)amt; - }else{ - sqlite3VdbeMemRelease(pMem); - } - } + rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem); } return rc; @@ -62745,7 +66066,7 @@ struct ValueNewStat4Ctx { ** Otherwise, if the second argument is non-zero, then this function is ** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not ** already been allocated, allocate the UnpackedRecord structure that -** that function will return to its caller here. Then return a pointer +** that function will return to its caller here. Then return a pointer to ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ @@ -62789,6 +66110,113 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ return sqlite3ValueNew(db); } +/* +** The expression object indicated by the second argument is guaranteed +** to be a scalar SQL function. If +** +** * all function arguments are SQL literals, +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and +** * the SQLITE_FUNC_NEEDCOLL function flag is not set, +** +** then this routine attempts to invoke the SQL function. Assuming no +** error occurs, output parameter (*ppVal) is set to point to a value +** object containing the result before returning SQLITE_OK. +** +** Affinity aff is applied to the result of the function before returning. +** If the result is a text value, the sqlite3_value object uses encoding +** enc. +** +** If the conditions above are not met, this function returns SQLITE_OK +** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to +** NULL and an SQLite error code returned. +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static int valueFromFunction( + sqlite3 *db, /* The database connection */ + Expr *p, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 aff, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + struct ValueNewStat4Ctx *pCtx /* Second argument for valueNew() */ +){ + sqlite3_context ctx; /* Context object for function invocation */ + sqlite3_value **apVal = 0; /* Function arguments */ + int nVal = 0; /* Size of apVal[] array */ + FuncDef *pFunc = 0; /* Function definition */ + sqlite3_value *pVal = 0; /* New value */ + int rc = SQLITE_OK; /* Return code */ + int nName; /* Size of function name in bytes */ + ExprList *pList = 0; /* Function arguments */ + int i; /* Iterator variable */ + + assert( pCtx!=0 ); + assert( (p->flags & EP_TokenOnly)==0 ); + pList = p->x.pList; + if( pList ) nVal = pList->nExpr; + nName = sqlite3Strlen30(p->u.zToken); + pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0); + assert( pFunc ); + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + ){ + return SQLITE_OK; + } + + if( pList ){ + apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal); + if( apVal==0 ){ + rc = SQLITE_NOMEM; + goto value_from_function_out; + } + for(i=0; ia[i].pExpr, enc, aff, &apVal[i]); + if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out; + } + } + + pVal = valueNew(db, pCtx); + if( pVal==0 ){ + rc = SQLITE_NOMEM; + goto value_from_function_out; + } + + assert( pCtx->pParse->rc==SQLITE_OK ); + memset(&ctx, 0, sizeof(ctx)); + ctx.pOut = pVal; + ctx.pFunc = pFunc; + pFunc->xFunc(&ctx, nVal, apVal); + if( ctx.isError ){ + rc = ctx.isError; + sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); + }else{ + sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); + assert( rc==SQLITE_OK ); + rc = sqlite3VdbeChangeEncoding(pVal, enc); + if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + rc = SQLITE_TOOBIG; + pCtx->pParse->nErr++; + } + } + pCtx->pParse->rc = rc; + + value_from_function_out: + if( rc!=SQLITE_OK ){ + pVal = 0; + } + if( apVal ){ + for(i=0; iop)==TK_UPLUS ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + /* Compressed expressions only appear when parsing the DEFAULT clause + ** on a table column definition, and hence only when pCtx==0. This + ** check ensures that an EP_TokenOnly expression is never passed down + ** into valueFromFunction(). */ + assert( (pExpr->flags & EP_TokenOnly)==0 || pCtx==0 ); + if( op==TK_CAST ){ u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); @@ -62853,7 +66287,7 @@ static int valueFromExpr( if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); @@ -62897,6 +66331,12 @@ static int valueFromExpr( } #endif +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + else if( op==TK_FUNCTION && pCtx!=0 ){ + rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); + } +#endif + *ppVal = pVal; return rc; @@ -63183,7 +66623,7 @@ SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipKeyInfo); sqlite3DbFree(db, pRec); @@ -63214,19 +66654,28 @@ SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ } /* -** Return the number of bytes in the sqlite3_value object assuming -** that it uses the encoding "enc" +** The sqlite3ValueBytes() routine returns the number of bytes in the +** sqlite3_value object assuming that it uses the encoding "enc". +** The valueBytes() routine is a helper function. */ +static SQLITE_NOINLINE int valueBytes(sqlite3_value *pVal, u8 enc){ + return valueToText(pVal, enc)!=0 ? pVal->n : 0; +} SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ Mem *p = (Mem*)pVal; - if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ + assert( (p->flags & MEM_Null)==0 || (p->flags & (MEM_Str|MEM_Blob))==0 ); + if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ + return p->n; + } + if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; }else{ return p->n; } } - return 0; + if( p->flags & MEM_Null ) return 0; + return valueBytes(pVal, enc); } /************** End of vdbemem.c *********************************************/ @@ -63245,6 +66694,8 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ ** This file contains code used for creating, destroying, and populating ** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Create a new virtual database engine. @@ -63269,6 +66720,17 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ return p; } +/* +** Change the error string stored in Vdbe.zErrMsg +*/ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ + va_list ap; + sqlite3DbFree(p->db, p->zErrMsg); + va_start(ap, zFormat); + p->zErrMsg = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); +} + /* ** Remember the SQL string for a prepared statement. */ @@ -63286,9 +66748,9 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepa /* ** Return the SQL associated with a prepared statement */ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return (p && p->isPrepareV2) ? p->zSql : 0; + return p ? p->zSql : 0; } /* @@ -63435,6 +66897,44 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } +/* Generate code for an unconditional jump to instruction iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ + return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); +} + +/* Generate code to cause the string zStr to be loaded into +** register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ + return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); +} + +/* +** Generate code that initializes multiple registers to string or integer +** constants. The registers begin with iDest and increase consecutively. +** One register is initialized for each characgter in zTypes[]. For each +** "s" character in zTypes[], the register is a string if the argument is +** not NULL, or OP_Null if the value is a null pointer. For each "i" character +** in zTypes[], the register is initialized to an integer. +*/ +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ + va_list ap; + int i; + char c; + va_start(ap, zTypes); + for(i=0; (c = zTypes[i])!=0; i++){ + if( c=='s' ){ + const char *z = va_arg(ap, const char*); + int addr = sqlite3VdbeAddOp2(p, z==0 ? OP_Null : OP_String8, 0, iDest++); + if( z ) sqlite3VdbeChangeP4(p, addr, z, 0); + }else{ + assert( c=='i' ); + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); + } + } + va_end(ap); +} /* ** Add an opcode that includes the p4 value as a pointer. @@ -63453,6 +66953,24 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an opcode that includes the p4 value with a P4_INT64 or +** P4_REAL type. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const u8 *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + char *p4copy = sqlite3DbMallocRaw(sqlite3VdbeDb(p), 8); + if( p4copy ) memcpy(p4copy, zP4, 8); + return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); +} + /* ** Add an OP_ParseSchema opcode. This routine is broken out from ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees @@ -63522,7 +67040,8 @@ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ int j = -1-x; assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - if( ALWAYS(j>=0) && p->aLabel ){ + assert( j>=0 ); + if( p->aLabel ){ p->aLabel[j] = v->nOp; } p->iFixedOp = v->nOp - 1; @@ -63617,6 +67136,7 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_VUpdate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does @@ -63627,6 +67147,9 @@ static Op *opIterNext(VdbeOpIter *p){ */ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; + int hasFkCounter = 0; + int hasCreateTable = 0; + int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -63635,15 +67158,19 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename -#ifndef SQLITE_OMIT_FOREIGN_KEY - || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) -#endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } + if( opcode==OP_CreateTable ) hasCreateTable = 1; + if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ + hasFkCounter = 1; + } +#endif } sqlite3DbFree(v->db, sIter.apSub); @@ -63652,22 +67179,27 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter + || (hasCreateTable && hasInitCoroutine) ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** Loop through the program looking for P2 values that are negative -** on jump instructions. Each such value is a label. Resolve the -** label by setting the P2 value to its correct non-zero value. +** This routine is called after all opcodes have been inserted. It loops +** through all the opcodes and fixes up some details. +** +** (1) For each jump instruction with a negative P2 value (a label) +** resolve the P2 value to an actual address. ** -** This routine is called once after all opcodes have been inserted. +** (2) Compute the maximum number of arguments used by any SQL function +** and store that value in *pMaxFuncArgs. ** -** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument -** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by -** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately +** indicate what the prepared statement actually does. ** -** The Op.opflags field is set on all opcodes. +** (4) Initialize the p4.xAdvance pointer on opcodes that use it. +** +** (5) Reclaim the memory allocated for storing labels. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ int i; @@ -63683,11 +67215,6 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ /* NOTE: Be sure to update mkopcodeh.awk when adding or removing ** cases from this switch! */ switch( opcode ){ - case OP_Function: - case OP_AggStep: { - if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; - break; - } case OP_Transaction: { if( pOp->p2!=0 ) p->readOnly = 0; /* fall thru */ @@ -63785,93 +67312,93 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) ** address of the first operation added. */ SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ - int addr; + int addr, i; + VdbeOp *pOut; + assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } addr = p->nOp; - if( ALWAYS(nOp>0) ){ - int i; - VdbeOpList const *pIn = aOp; - for(i=0; ip2; - VdbeOp *pOut = &p->aOp[i+addr]; - pOut->opcode = pIn->opcode; - pOut->p1 = pIn->p1; - if( p2<0 ){ - assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); - pOut->p2 = addr + ADDR(p2); - }else{ - pOut->p2 = p2; - } - pOut->p3 = pIn->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; + pOut = &p->aOp[addr]; + for(i=0; ip2; + pOut->opcode = aOp->opcode; + pOut->p1 = aOp->p1; + if( p2<0 ){ + assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); + pOut->p2 = addr + ADDR(p2); + }else{ + pOut->p2 = p2; + } + pOut->p3 = aOp->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOut->zComment = 0; + pOut->zComment = 0; #endif #ifdef SQLITE_VDBE_COVERAGE - pOut->iSrcLine = iLineno+i; + pOut->iSrcLine = iLineno+i; #else - (void)iLineno; + (void)iLineno; #endif #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); - } -#endif + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } - p->nOp += nOp; +#endif } + p->nOp += nOp; return addr; } +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* -** Change the value of the P1 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). */ -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p1 = val; +SQLITE_PRIVATE void sqlite3VdbeScanStatus( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + LogEst nEst, /* Estimated number of output rows */ + const char *zName /* Name of table or index being scanned */ +){ + int nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; } } +#endif + /* -** Change the value of the P2 operand for a specific instruction. -** This routine is useful for setting a jump destination. +** Change the value of the opcode, or P1, P2, P3, or P5 operands +** for a specific instruction. */ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; +} +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ + sqlite3VdbeGetOp(p,addr)->p1 = val; +} SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p2 = val; - } + sqlite3VdbeGetOp(p,addr)->p2 = val; } - -/* -** Change the value of the P3 operand for a specific instruction. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p3 = val; - } + sqlite3VdbeGetOp(p,addr)->p3 = val; } - -/* -** Change the value of the P5 operand for the most recently -** added operation. -*/ -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ - assert( p!=0 ); - if( p->aOp ){ - assert( p->nOp>0 ); - p->aOp[p->nOp-1].p5 = val; - } +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ + sqlite3VdbeGetOp(p,-1)->p5 = p5; } /* @@ -63879,8 +67406,8 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ ** the address of the next instruction to be coded. */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - sqlite3VdbeChangeP2(p, addr, p->nOp); p->pParse->iFixedOp = p->nOp - 1; + sqlite3VdbeChangeP2(p, addr, p->nOp); } @@ -63903,6 +67430,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ + case P4_FUNCCTX: { + freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc); + /* Fall through into the next case */ + } case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -64261,8 +67792,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zColl = "B"; n = 1; } - if( i+n>nTemp-6 ){ + if( i+n>nTemp-7 ){ memcpy(&zTemp[i],",...",4); + i += 4; break; } zTemp[i++] = ','; @@ -64287,6 +67819,13 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); break; } +#ifdef SQLITE_DEBUG + case P4_FUNCCTX: { + FuncDef *pDef = pOp->p4.pCtx->pFunc; + sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + break; + } +#endif case P4_INT64: { sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); break; @@ -64318,7 +67857,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ #ifndef SQLITE_OMIT_VIRTUALTABLE case P4_VTAB: { sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3_snprintf(nTemp, zTemp, "vtab:%p:%p", pVtab, pVtab->pModule); + sqlite3_snprintf(nTemp, zTemp, "vtab:%p", pVtab); break; } #endif @@ -64407,12 +67946,11 @@ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ /* ** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). */ -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ +static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ int i; sqlite3 *db; Db *aDb; int nDb; - if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; @@ -64422,6 +67960,10 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ } } } +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + vdbeLeave(p); +} #endif #if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) @@ -64594,7 +68136,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( }else if( db->u1.isInterrupted ){ p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); }else{ char *zP4; Op *pOp; @@ -64926,6 +68468,9 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); +#endif if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -64942,7 +68487,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar[n].db = db; } } - if( p->azVar ){ + if( p->azVar && pParse->nzVar>0 ){ p->nzVar = pParse->nzVar; memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); @@ -64979,13 +68524,29 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ else if( pCx->pVtabCursor ){ sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; - p->inVtabMethod = 1; + assert( pVtabCursor->pVtab->nRef>0 ); + pVtabCursor->pVtab->nRef--; pModule->xClose(pVtabCursor); - p->inVtabMethod = 0; } #endif } +/* +** Close all cursors in the current frame. +*/ +static void closeCursorsInFrame(Vdbe *p){ + if( p->apCsr ){ + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursor(p, pC); + p->apCsr[i] = 0; + } + } + } +} + /* ** Copy the values stored in the VdbeFrame structure to its Vdbe. This ** is used, for example, when a trigger sub-program is halted to restore @@ -64993,6 +68554,10 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; + closeCursorsInFrame(v); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + v->anExec = pFrame->anExec; +#endif v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; @@ -65003,6 +68568,7 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; return pFrame->pc; } @@ -65023,17 +68589,7 @@ static void closeAllCursors(Vdbe *p){ p->nFrame = 0; } assert( p->nFrame==0 ); - - if( p->apCsr ){ - int i; - for(i=0; inCursor; i++){ - VdbeCursor *pC = p->apCsr[i]; - if( pC ){ - sqlite3VdbeFreeCursor(p, pC); - p->apCsr[i] = 0; - } - } - } + closeCursorsInFrame(p); if( p->aMem ){ releaseMemArray(&p->aMem[1], p->nMem); } @@ -65336,7 +68892,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** doing this the directory is synced again before any individual ** transaction files are deleted. */ - rc = sqlite3OsDelete(pVfs, zMaster, 1); + rc = sqlite3OsDelete(pVfs, zMaster, needSync); sqlite3DbFree(db, zMaster); zMaster = 0; if( rc ){ @@ -65483,7 +69039,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ ){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; - sqlite3SetString(&p->zErrMsg, db, "FOREIGN KEY constraint failed"); + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); return SQLITE_ERROR; } return SQLITE_OK; @@ -65570,6 +69126,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -65610,6 +69167,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -65618,6 +69176,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -65629,6 +69188,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -65649,6 +69209,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -65910,6 +69471,12 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); +#endif } /* @@ -66068,9 +69635,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ i64 i = pMem->u.i; u64 u; if( i<0 ){ - if( i<(-MAX_6BYTE) ) return 6; - /* Previous test prevents: u = -(-9223372036854775808) */ - u = -i; + u = ~i; }else{ u = i; } @@ -66095,6 +69660,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +/* +** The sizes for serial types less than 12 +*/ +static const u8 sqlite3SmallTypeSizes[] = { + 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 +}; + /* ** Return the length of the data corresponding to the supplied serial-type. */ @@ -66102,8 +69674,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ if( serial_type>=12 ){ return (serial_type-12)/2; }else{ - static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; - return aSize[serial_type]; + return sqlite3SmallTypeSizes[serial_type]; } } @@ -66187,7 +69758,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ }else{ v = pMem->u.i; } - len = i = sqlite3VdbeSerialTypeLen(serial_type); + len = i = sqlite3SmallTypeSizes[serial_type]; assert( i>0 ); do{ buf[--i] = (u8)(v&0xFF); @@ -66236,10 +69807,14 @@ static u32 SQLITE_NOINLINE serialGet( u32 y = FOUR_BYTE_UINT(buf+4); x = (x<<32) + y; if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ #if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) /* Verify that integers and floating point values use the same ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is @@ -66267,35 +69842,46 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ - case 0: { /* NULL */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; break; } - case 1: { /* 1-byte signed integer */ + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); @@ -66309,11 +69895,17 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; return 0; } default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; @@ -66451,6 +70043,7 @@ static int vdbeRecordCompareDebug( /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); + if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pKeyInfo->aSortOrder!=0 ); @@ -66512,6 +70105,42 @@ static int vdbeRecordCompareDebug( } #endif +#if SQLITE_DEBUG +/* +** Count the number of fields (a.k.a. columns) in the record given by +** pKey,nKey. The verify that this count is less than or equal to the +** limit given by pKeyInfo->nField + pKeyInfo->nXField. +** +** If this constraint is not satisfied, it means that the high-speed +** vdbeRecordCompareInt() and vdbeRecordCompareString() routines will +** not work correctly. If this assert() ever fires, it probably means +** that the KeyInfo.nField or KeyInfo.nXField values were computed +** incorrectly. +*/ +static void vdbeAssertFieldCountWithinLimits( + int nKey, const void *pKey, /* The record to verify */ + const KeyInfo *pKeyInfo /* Compare size with this KeyInfo */ +){ + int nField = 0; + u32 szHdr; + u32 idx; + u32 notUsed; + const unsigned char *aKey = (const unsigned char*)pKey; + + if( CORRUPT_DB ) return; + idx = getVarint32(aKey, szHdr); + assert( nKey>=0 ); + assert( szHdr<=(u32)nKey ); + while( idxnField+pKeyInfo->nXField ); +} +#else +# define vdbeAssertFieldCountWithinLimits(A,B,C) +#endif + /* ** Both *pMem1 and *pMem2 contain string values. Compare the two values ** using the collation sequence pColl. As usual, return a negative , zero @@ -66712,7 +70341,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ -static int vdbeRecordCompareWithSkip( +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ @@ -66760,7 +70389,7 @@ static int vdbeRecordCompareWithSkip( if( pRhs->flags & MEM_Int ){ serial_type = aKey1[idx1]; testcase( serial_type==12 ); - if( serial_type>=12 ){ + if( serial_type>=10 ){ rc = +1; }else if( serial_type==0 ){ rc = -1; @@ -66786,7 +70415,11 @@ static int vdbeRecordCompareWithSkip( /* RHS is real */ else if( pRhs->flags & MEM_Real ){ serial_type = aKey1[idx1]; - if( serial_type>=12 ){ + if( serial_type>=10 ){ + /* Serial types 12 or greater are strings and blobs (greater than + ** numbers). Types 10 and 11 are currently "reserved for future + ** use", so it doesn't really matter what the results of comparing + ** them to numberic values are. */ rc = +1; }else if( serial_type==0 ){ rc = -1; @@ -66898,7 +70531,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2 /* Right key */ ){ - return vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); } @@ -66923,6 +70556,7 @@ static int vdbeRecordCompareInt( i64 v = pPKey2->aMem[0].u.i; i64 lhs; + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ @@ -66985,7 +70619,7 @@ static int vdbeRecordCompareInt( }else if( pPKey2->nField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ @@ -67010,6 +70644,7 @@ static int vdbeRecordCompareString( int serial_type; int res; + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); getVarint32(&aKey1[1], serial_type); if( serial_type<12 ){ res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ @@ -67032,7 +70667,7 @@ static int vdbeRecordCompareString( res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; } @@ -67153,7 +70788,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ goto idx_rowid_corruption; } - lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); + lenRowid = sqlite3SmallTypeSizes[typeRowid]; testcase( (u32)m.n==szHdr+lenRowid ); if( unlikely((u32)m.nexpired; } @@ -67364,6 +71001,31 @@ static int vdbeSafetyNotNull(Vdbe *p){ } } +#ifndef SQLITE_OMIT_TRACE +/* +** Invoke the profile callback. This routine is only called if we already +** know that the profile callback is defined and needs to be invoked. +*/ +static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ + sqlite3_int64 iNow; + assert( p->startTime>0 ); + assert( db->xProfile!=0 ); + assert( db->init.busy==0 ); + assert( p->zSql!=0 ); + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + p->startTime = 0; +} +/* +** The checkProfileCallback(DB,P) macro checks to see if a profile callback +** is needed, and it invokes the callback if it is needed. +*/ +# define checkProfileCallback(DB,P) \ + if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); } +#else +# define checkProfileCallback(DB,P) /*no-op*/ +#endif + /* ** The following routine destroys a virtual machine that is created by ** the sqlite3_compile() routine. The integer returned is an SQLITE_ @@ -67373,7 +71035,7 @@ static int vdbeSafetyNotNull(Vdbe *p){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL @@ -67384,6 +71046,7 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ sqlite3 *db = v->db; if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeFinalize(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); @@ -67399,18 +71062,20 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; - sqlite3_mutex_enter(v->db->mutex); + sqlite3 *db = v->db; + sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeReset(v); sqlite3VdbeRewind(v); - assert( (rc & (v->db->errMask))==rc ); - rc = sqlite3ApiExit(v->db, rc); - sqlite3_mutex_leave(v->db->mutex); + assert( (rc & (db->errMask))==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); } return rc; } @@ -67418,7 +71083,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ /* ** Set all the parameters in the compiled SQL statement to NULL. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int i; int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; @@ -67442,46 +71107,56 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ ** The following routines extract information from a Mem or sqlite3_value ** structure. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - sqlite3VdbeMemExpandBlob(p); + if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ){ + assert( p->flags==MEM_Null && p->z==0 ); + return 0; + } p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ return sqlite3_value_text(pVal); } } -SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF8); } -SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ +SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value *pVal){ return sqlite3VdbeRealValue((Mem*)pVal); } -SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value *pVal){ return (int)sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ +SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value *pVal){ + return ((Mem*)pVal)->eSubtype; +} +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value* pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16BE); } -SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16LE); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ +/* EVIDENCE-OF: R-12793-43283 Every value in SQLite has one of five +** fundamental datatypes: 64-bit signed integer 64-bit IEEE floating +** point number string BLOB NULL +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { SQLITE_BLOB, /* 0x00 */ SQLITE_NULL, /* 0x01 */ @@ -67519,6 +71194,36 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ return aType[pVal->flags&MEM_AffMask]; } +/* Make a copy of an sqlite3_value object +*/ +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value *pOrig){ + sqlite3_value *pNew; + if( pOrig==0 ) return 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + memcpy(pNew, pOrig, MEMCELLSIZE); + pNew->flags &= ~MEM_Dyn; + pNew->db = 0; + if( pNew->flags&(MEM_Str|MEM_Blob) ){ + pNew->flags &= ~(MEM_Static|MEM_Dyn); + pNew->flags |= MEM_Ephem; + if( sqlite3VdbeMemMakeWriteable(pNew)!=SQLITE_OK ){ + sqlite3ValueFree(pNew); + pNew = 0; + } + } + return pNew; +} + +/* Destroy an sqlite3_value object previously obtained from +** sqlite3_value_dup(). +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value *pOld){ + sqlite3ValueFree(pOld); +} + + /**************************** sqlite3_result_ ******************************* ** The following routines are used by user-defined functions to specify ** the function result. @@ -67557,7 +71262,7 @@ static int invokeValueDestructor( if( pCtx ) sqlite3_result_error_toobig(pCtx); return SQLITE_TOOBIG; } -SQLITE_API void sqlite3_result_blob( +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob( sqlite3_context *pCtx, const void *z, int n, @@ -67567,7 +71272,7 @@ SQLITE_API void sqlite3_result_blob( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } -SQLITE_API void sqlite3_result_blob64( +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64( sqlite3_context *pCtx, const void *z, sqlite3_uint64 n, @@ -67581,37 +71286,41 @@ SQLITE_API void sqlite3_result_blob64( setResultStrOrError(pCtx, z, (int)n, 0, xDel); } } -SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context *pCtx, double rVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } -SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif -SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context *pCtx, int iVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } -SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } -SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } -SQLITE_API void sqlite3_result_text( +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->pOut->eSubtype = eSubtype & 0xff; +} +SQLITE_API void SQLITE_STDCALL sqlite3_result_text( sqlite3_context *pCtx, const char *z, int n, @@ -67620,7 +71329,7 @@ SQLITE_API void sqlite3_result_text( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } -SQLITE_API void sqlite3_result_text64( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text64( sqlite3_context *pCtx, const char *z, sqlite3_uint64 n, @@ -67637,7 +71346,7 @@ SQLITE_API void sqlite3_result_text64( } } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_text16( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16( sqlite3_context *pCtx, const void *z, int n, @@ -67646,7 +71355,7 @@ SQLITE_API void sqlite3_result_text16( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); } -SQLITE_API void sqlite3_result_text16be( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be( sqlite3_context *pCtx, const void *z, int n, @@ -67655,7 +71364,7 @@ SQLITE_API void sqlite3_result_text16be( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); } -SQLITE_API void sqlite3_result_text16le( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le( sqlite3_context *pCtx, const void *z, int n, @@ -67665,17 +71374,29 @@ SQLITE_API void sqlite3_result_text16le( setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemCopy(pCtx->pOut, pValue); } -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); } -SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ +SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + return SQLITE_TOOBIG; + } + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); + return SQLITE_OK; +} +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; +#ifdef SQLITE_DEBUG + if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; +#endif if( pCtx->pOut->flags & MEM_Null ){ sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, SQLITE_STATIC); @@ -67683,7 +71404,7 @@ SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ } /* Force an SQLITE_TOOBIG error. */ -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; pCtx->fErrorOrAux = 1; @@ -67692,7 +71413,7 @@ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ } /* An SQLITE_NOMEM error. */ -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM; @@ -67711,7 +71432,10 @@ static int doWalCallbacks(sqlite3 *db){ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + int nEntry; + sqlite3BtreeEnter(pBt); + nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); } @@ -67721,6 +71445,7 @@ static int doWalCallbacks(sqlite3 *db){ return rc; } + /* ** Execute the statement pStmt, either until a row of data is ready, the ** statement is completely executed or an error occurs. @@ -67753,7 +71478,7 @@ static int sqlite3Step(Vdbe *p){ ** or SQLITE_BUSY error. */ #ifdef SQLITE_OMIT_AUTORESET - if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){ + if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ sqlite3_reset((sqlite3_stmt*)p); }else{ return SQLITE_MISUSE_BKPT; @@ -67789,8 +71514,10 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( db->xProfile && !db->init.busy ){ + if( db->xProfile && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); } #endif @@ -67799,6 +71526,9 @@ static int sqlite3Step(Vdbe *p){ if( p->bIsReader ) db->nVdbeRead++; p->pc = 0; } +#ifdef SQLITE_DEBUG + p->rcApp = SQLITE_OK; +#endif #ifndef SQLITE_OMIT_EXPLAIN if( p->explain ){ rc = sqlite3VdbeList(p); @@ -67811,13 +71541,8 @@ static int sqlite3Step(Vdbe *p){ } #ifndef SQLITE_OMIT_TRACE - /* Invoke the profile callback if there is one - */ - if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ - sqlite3_int64 iNow; - sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); - } + /* If the statement completed successfully, invoke the profile callback */ + if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); #endif if( rc==SQLITE_DONE ){ @@ -67841,9 +71566,9 @@ static int sqlite3Step(Vdbe *p){ ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); - assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); + assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ /* If this statement was prepared using sqlite3_prepare_v2(), and an ** error has occurred, then return the error code in p->rc to the @@ -67859,7 +71584,7 @@ static int sqlite3Step(Vdbe *p){ ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. */ -SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ int rc = SQLITE_OK; /* Result from sqlite3Step() */ int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ @@ -67891,7 +71616,6 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** sqlite3_errmsg() and sqlite3_errcode(). */ const char *zErr = (const char *)sqlite3_value_text(db->pErr); - assert( zErr!=0 || db->mallocFailed ); sqlite3DbFree(db, v->zErrMsg); if( !db->mallocFailed ){ v->zErrMsg = sqlite3DbStrDup(db, zErr); @@ -67911,7 +71635,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ +SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ assert( p && p->pFunc ); return p->pFunc->pUserData; } @@ -67926,22 +71650,32 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ ** sqlite3_create_function16() routines that originally registered the ** application defined function. */ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p){ + assert( p && p->pOut ); return p->pOut->db; } /* -** Return the current time for a statement +** Return the current time for a statement. If the current time +** is requested more than once within the same run of a single prepared +** statement, the exact same time is returned for each invocation regardless +** of the amount of time that elapses between invocations. In other words, +** the time returned is always the time of the first call. */ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ - Vdbe *v = p->pVdbe; int rc; - if( v->iCurrentTime==0 ){ - rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, &v->iCurrentTime); - if( rc ) v->iCurrentTime = 0; +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; + assert( p->pVdbe!=0 ); +#else + sqlite3_int64 iTime = 0; + sqlite3_int64 *piTime = p->pVdbe!=0 ? &p->pVdbe->iCurrentTime : &iTime; +#endif + if( *piTime==0 ){ + rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, piTime); + if( rc ) *piTime = 0; } - return v->iCurrentTime; + return *piTime; } /* @@ -67992,7 +71726,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ ** context is allocated on the first call. Subsequent calls return the ** same context that was returned on prior calls. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ +SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context *p, int nByte){ assert( p && p->pFunc && p->pFunc->xStep ); assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); @@ -68007,10 +71741,15 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ ** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ +SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +#if SQLITE_ENABLE_STAT3_OR_STAT4 + if( pCtx->pVdbe==0 ) return 0; +#else + assert( pCtx->pVdbe!=0 ); +#endif for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; } @@ -68023,7 +71762,7 @@ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. */ -SQLITE_API void sqlite3_set_auxdata( +SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata( sqlite3_context *pCtx, int iArg, void *pAux, @@ -68034,6 +71773,11 @@ SQLITE_API void sqlite3_set_auxdata( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( iArg<0 ) goto failed; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pVdbe==0 ) goto failed; +#else + assert( pVdbe!=0 ); +#endif for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; @@ -68073,7 +71817,7 @@ SQLITE_API void sqlite3_set_auxdata( ** implementations should keep their own counts within their aggregate ** context. */ -SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context *p){ assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); return p->pMem->n; } @@ -68082,7 +71826,7 @@ SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ /* ** Return the number of columns in the result set for the statement pStmt. */ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; return pVm ? pVm->nResColumn : 0; } @@ -68091,7 +71835,7 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ ** Return the number of values available from the current row of the ** currently executing statement pStmt. */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; if( pVm==0 || pVm->pResultSet==0 ) return 0; return pVm->nResColumn; @@ -68116,18 +71860,19 @@ static const Mem *columnNullValue(void){ #endif = { /* .u = */ {0}, - /* .flags = */ MEM_Null, - /* .enc = */ 0, - /* .n = */ 0, - /* .z = */ 0, - /* .zMalloc = */ 0, - /* .szMalloc = */ 0, - /* .iPadding1 = */ 0, - /* .db = */ 0, - /* .xDel = */ 0, + /* .flags = */ (u16)MEM_Null, + /* .enc = */ (u8)0, + /* .eSubtype = */ (u8)0, + /* .n = */ (int)0, + /* .z = */ (char*)0, + /* .zMalloc = */ (char*)0, + /* .szMalloc = */ (int)0, + /* .uTemp = */ (u32)0, + /* .db = */ (sqlite3*)0, + /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG - /* .pScopyFrom = */ 0, - /* .pFiller = */ 0, + /* .pScopyFrom = */ (Mem*)0, + /* .pFiller = */ (void*)0, #endif }; return &nullMem; @@ -68193,7 +71938,7 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) ** The following routines are used to access elements of the current row ** in the result set. */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ const void *val; val = sqlite3_value_blob( columnMem(pStmt,i) ); /* Even though there is no encoding conversion, value_blob() might @@ -68203,37 +71948,37 @@ SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ +SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt *pStmt, int i){ double val = sqlite3_value_double( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_int( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt *pStmt, int i){ const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt *pStmt, int i){ Mem *pOut = columnMem(pStmt, i); if( pOut->flags&MEM_Static ){ pOut->flags &= ~MEM_Static; @@ -68243,13 +71988,13 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ return (sqlite3_value *)pOut; } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt *pStmt, int i){ int iType = sqlite3_value_type( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return iType; @@ -68277,11 +72022,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -68305,12 +72058,12 @@ static const void *columnName( ** Return the name of the Nth column of the result set returned by SQL ** statement pStmt. */ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); } @@ -68330,12 +72083,12 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ ** Return the column declaration type (if applicable) of the 'i'th column ** of the result set of SQL statement pStmt. */ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); } @@ -68348,12 +72101,12 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); } @@ -68364,12 +72117,12 @@ SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); } @@ -68380,12 +72133,12 @@ SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); } @@ -68486,7 +72239,7 @@ static int bindText( /* ** Bind a blob value to an SQL statement variable. */ -SQLITE_API int sqlite3_bind_blob( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob( sqlite3_stmt *pStmt, int i, const void *zData, @@ -68495,7 +72248,7 @@ SQLITE_API int sqlite3_bind_blob( ){ return bindText(pStmt, i, zData, nData, xDel, 0); } -SQLITE_API int sqlite3_bind_blob64( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64( sqlite3_stmt *pStmt, int i, const void *zData, @@ -68509,7 +72262,7 @@ SQLITE_API int sqlite3_bind_blob64( return bindText(pStmt, i, zData, (int)nData, xDel, 0); } } -SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -68519,10 +72272,10 @@ SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ } return rc; } -SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ return sqlite3_bind_int64(p, i, (i64)iValue); } -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -68532,7 +72285,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu } return rc; } -SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; rc = vdbeUnbind(p, i); @@ -68541,7 +72294,7 @@ SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ } return rc; } -SQLITE_API int sqlite3_bind_text( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text( sqlite3_stmt *pStmt, int i, const char *zData, @@ -68550,7 +72303,7 @@ SQLITE_API int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } -SQLITE_API int sqlite3_bind_text64( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64( sqlite3_stmt *pStmt, int i, const char *zData, @@ -68567,7 +72320,7 @@ SQLITE_API int sqlite3_bind_text64( } } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_bind_text16( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, @@ -68577,7 +72330,7 @@ SQLITE_API int sqlite3_bind_text16( return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; switch( sqlite3_value_type((sqlite3_value*)pValue) ){ case SQLITE_INTEGER: { @@ -68608,7 +72361,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu } return rc; } -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -68618,12 +72371,26 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ } return rc; } +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + rc = SQLITE_TOOBIG; + }else{ + assert( (n & 0x7FFFFFFF)==n ); + rc = sqlite3_bind_zeroblob(pStmt, i, n); + } + rc = sqlite3ApiExit(p->db, rc); + sqlite3_mutex_leave(p->db->mutex); + return rc; +} /* ** Return the number of wildcards that can be potentially bound to. ** This routine is added to support DBD::SQLite. */ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; return p ? p->nVar : 0; } @@ -68634,7 +72401,7 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ ** ** The result is always UTF-8. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; if( p==0 || i<1 || i>p->nzVar ){ return 0; @@ -68662,7 +72429,7 @@ SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nNa } return 0; } -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); } @@ -68696,7 +72463,7 @@ SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt ** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise ** SQLITE_OK is returned. */ -SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ Vdbe *pFrom = (Vdbe*)pFromStmt; Vdbe *pTo = (Vdbe*)pToStmt; if( pFrom->nVar!=pTo->nVar ){ @@ -68718,7 +72485,7 @@ SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt * ** the first argument to the sqlite3_prepare() that was used to create ** the statement in the first place. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->db : 0; } @@ -68726,14 +72493,14 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ ** Return true if the prepared statement is guaranteed to not modify the ** database. */ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } /* ** Return true if the prepared statement is in need of being reset. */ -SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; } @@ -68744,8 +72511,14 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ ** prepared statement for the database connection. Return NULL if there ** are no more. */ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ +SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -68759,13 +72532,89 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ /* ** Return the value of a status counter for a prepared statement */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + Vdbe *p = (Vdbe*)pStmt; + ScanStatus *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName; + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + default: { + return 1; + } + } + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + memset(p->anExec, 0, p->nOp * sizeof(i64)); +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + /************** End of vdbeapi.c *********************************************/ /************** Begin file vdbetrace.c ***************************************/ /* @@ -68785,6 +72634,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ ** ** The Vdbe parse-tree explainer is also found here. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_TRACE @@ -68852,9 +72703,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( char zBase[100]; /* Initial working space */ db = p->db; - sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - out.db = db; if( db->nVdbeExec>1 ){ while( *zRawSql ){ const char *zStart = zRawSql; @@ -68863,6 +72713,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( assert( (zRawSql - zStart) > 0 ); sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart)); } + }else if( p->nVar==0 ){ + sqlite3StrAccumAppend(&out, zRawSql, sqlite3Strlen30(zRawSql)); }else{ while( zRawSql[0] ){ n = findNextHostParameter(zRawSql, &nToken); @@ -68879,10 +72731,12 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( idx = nextIndex; } }else{ - assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); + assert( zRawSql[0]==':' || zRawSql[0]=='$' || + zRawSql[0]=='@' || zRawSql[0]=='#' ); testcase( zRawSql[0]==':' ); testcase( zRawSql[0]=='$' ); testcase( zRawSql[0]=='@' ); + testcase( zRawSql[0]=='#' ); idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); assert( idx>0 ); } @@ -68974,6 +72828,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Invoke this macro on memory cells just prior to changing the @@ -69224,7 +73080,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** -** SQLITE_AFF_NONE: +** SQLITE_AFF_BLOB: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -69250,6 +73106,7 @@ static void applyAffinity( if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ sqlite3VdbeMemStringify(pRec, enc, 1); } + pRec->flags &= ~(MEM_Real|MEM_Int); } } @@ -69259,7 +73116,7 @@ static void applyAffinity( ** is appropriate. But only do the conversion if it is possible without ** loss of information and return the revised type of the argument. */ -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; @@ -69557,6 +73414,21 @@ static int checkSavepointCount(sqlite3 *db){ } #endif +/* +** Return the register of pOp->p2 after first preparing it to be +** overwritten with an integer value. +*/ +static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ + Mem *pOut; + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + pOut = &p->aMem[pOp->p2]; + memAboutToChange(p, pOut); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; +} + /* ** Execute as much of a VDBE program as we can. @@ -69565,9 +73437,11 @@ static int checkSavepointCount(sqlite3 *db){ SQLITE_PRIVATE int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ - int pc=0; /* The program counter */ Op *aOp = p->aOp; /* Copy of p->aOp */ - Op *pOp; /* Current operation */ + Op *pOp = aOp; /* Current operation */ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + Op *pOrigOp; /* Value of pOp at the top of the loop */ +#endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ @@ -69596,7 +73470,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -69607,13 +73481,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( sqlite3VdbeIOTraceSql(p); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; assert( 0 < db->nProgressOps ); - nProgressLimit = (unsigned)p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - if( nProgressLimit==0 ){ - nProgressLimit = db->nProgressOps; - }else{ - nProgressLimit %= (unsigned)db->nProgressOps; - } + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); } #endif #ifdef SQLITE_DEBUG @@ -69643,20 +73513,22 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } sqlite3EndBenignMalloc(); #endif - for(pc=p->pc; rc==SQLITE_OK; pc++){ - assert( pc>=0 && pcnOp ); + for(pOp=&aOp[p->pc]; rc==SQLITE_OK; pOp++){ + assert( pOp>=aOp && pOp<&aOp[p->nOp]); if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE start = sqlite3Hwtime(); #endif nVmStep++; - pOp = &aOp[pc]; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; +#endif /* Only allow tracing if SQLITE_DEBUG is defined. */ #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ - sqlite3VdbePrintOp(stdout, pc, pOp); + sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp); } #endif @@ -69673,23 +73545,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } #endif - /* On any opcode with the "out2-prerelease" tag, free any - ** external allocations out of mem[p2] and set mem[p2] to be - ** an undefined integer. Opcodes will either fill in the integer - ** value or convert mem[p2] to a different type. - */ - assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); - if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); - pOut->flags = MEM_Int; - } - /* Sanity checking on other operands */ #ifdef SQLITE_DEBUG + assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); assert( pOp->p1<=(p->nMem-p->nCursor) ); @@ -69722,6 +73580,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( memAboutToChange(p, &aMem[pOp->p3]); } #endif +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + pOrigOp = pOp; +#endif switch( pOp->opcode ){ @@ -69745,7 +73606,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** ** Other keywords in the comment that follows each case are used to ** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. -** Keywords include: in1, in2, in3, out2_prerelease, out2, out3. See +** Keywords include: in1, in2, in3, out2, out3. See ** the mkopcodeh.awk script for additional information. ** ** Documentation about VDBE opcodes is generated by scanning this file @@ -69773,7 +73634,8 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ - pc = pOp->p2 - 1; +jump_to_p2_and_check_for_interrupt: + pOp = &aOp[pOp->p2 - 1]; /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon @@ -69818,9 +73680,13 @@ case OP_Gosub: { /* jump */ assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; - pIn1->u.i = pc; + pIn1->u.i = (int)(pOp-aOp); REGISTER_TRACE(pOp->p1, pIn1); - pc = pOp->p2 - 1; + + /* Most jump operations do a goto to this spot in order to update + ** the pOp pointer. */ +jump_to_p2: + pOp = &aOp[pOp->p2 - 1]; break; } @@ -69832,7 +73698,7 @@ case OP_Gosub: { /* jump */ case OP_Return: { /* in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags==MEM_Int ); - pc = (int)pIn1->u.i; + pOp = &aOp[pIn1->u.i]; pIn1->flags = MEM_Undefined; break; } @@ -69856,7 +73722,7 @@ case OP_InitCoroutine: { /* jump */ assert( !VdbeMemDynamic(pOut) ); pOut->u.i = pOp->p3 - 1; pOut->flags = MEM_Int; - if( pOp->p2 ) pc = pOp->p2 - 1; + if( pOp->p2 ) goto jump_to_p2; break; } @@ -69876,7 +73742,7 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); - pc = pCaller->p2 - 1; + pOp = &aOp[pCaller->p2 - 1]; pIn1->flags = MEM_Undefined; break; } @@ -69900,9 +73766,9 @@ case OP_Yield: { /* in1, jump */ assert( VdbeMemDynamic(pIn1)==0 ); pIn1->flags = MEM_Int; pcDest = (int)pIn1->u.i; - pIn1->u.i = pc; + pIn1->u.i = (int)(pOp - aOp); REGISTER_TRACE(pOp->p1, pIn1); - pc = pcDest; + pOp = &aOp[pcDest]; break; } @@ -69953,30 +73819,34 @@ case OP_HaltIfNull: { /* in3 */ case OP_Halt: { const char *zType; const char *zLogFmt; + VdbeFrame *pFrame; + int pcx; + pcx = (int)(pOp - aOp); if( pOp->p1==SQLITE_OK && p->pFrame ){ /* Halt the sub-program. Return control to the parent frame. */ - VdbeFrame *pFrame = p->pFrame; + pFrame = p->pFrame; p->pFrame = pFrame->pParent; p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); - pc = sqlite3VdbeFrameRestore(pFrame); + pcx = sqlite3VdbeFrameRestore(pFrame); lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ - /* Instruction pc is the OP_Program that invoked the sub-program + /* Instruction pcx is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt ** instruction is set to OE_Ignore, then the sub-program is throwing ** an IGNORE exception. In this case jump to the address specified ** as the p2 of the calling OP_Program. */ - pc = p->aOp[pc].p2-1; + pcx = p->aOp[pcx].p2-1; } aOp = p->aOp; aMem = p->aMem; + pOp = &aOp[pcx]; break; } p->rc = pOp->p1; p->errorAction = (u8)pOp->p2; - p->pc = pc; + p->pc = pcx; if( p->rc ){ if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", @@ -69993,14 +73863,13 @@ case OP_Halt: { assert( zType!=0 || pOp->p4.z!=0 ); zLogFmt = "abort at %d in [%s]: %s"; if( zType && pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed: %s", - zType, pOp->p4.z); + sqlite3VdbeError(p, "%s constraint failed: %s", zType, pOp->p4.z); }else if( pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + sqlite3VdbeError(p, "%s", pOp->p4.z); }else{ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed", zType); + sqlite3VdbeError(p, "%s constraint failed", zType); } - sqlite3_log(pOp->p1, zLogFmt, pc, p->zSql, p->zErrMsg); + sqlite3_log(pOp->p1, zLogFmt, pcx, p->zSql, p->zErrMsg); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -70019,7 +73888,8 @@ case OP_Halt: { ** ** The 32-bit integer value P1 is written into register P2. */ -case OP_Integer: { /* out2-prerelease */ +case OP_Integer: { /* out2 */ + pOut = out2Prerelease(p, pOp); pOut->u.i = pOp->p1; break; } @@ -70030,7 +73900,8 @@ case OP_Integer: { /* out2-prerelease */ ** P4 is a pointer to a 64-bit integer value. ** Write that value into register P2. */ -case OP_Int64: { /* out2-prerelease */ +case OP_Int64: { /* out2 */ + pOut = out2Prerelease(p, pOp); assert( pOp->p4.pI64!=0 ); pOut->u.i = *pOp->p4.pI64; break; @@ -70043,7 +73914,8 @@ case OP_Int64: { /* out2-prerelease */ ** P4 is a pointer to a 64-bit floating point value. ** Write that value into register P2. */ -case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ +case OP_Real: { /* same as TK_FLOAT, out2 */ + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Real; assert( !sqlite3IsNaN(*pOp->p4.pReal) ); pOut->u.r = *pOp->p4.pReal; @@ -70055,12 +73927,13 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ ** Synopsis: r[P2]='P4' ** ** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into a String before it is executed for the first time. During +** into a String opcode before it is executed for the first time. During ** this transformation, the length of string P4 is computed and stored ** as the P1 parameter. */ -case OP_String8: { /* same as TK_STRING, out2-prerelease */ +case OP_String8: { /* same as TK_STRING, out2 */ assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); pOp->opcode = OP_String; pOp->p1 = sqlite3Strlen30(pOp->p4.z); @@ -70087,18 +73960,31 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ /* Fall through to the next case, OP_String */ } -/* Opcode: String P1 P2 * P4 * +/* Opcode: String P1 P2 P3 P4 P5 ** Synopsis: r[P2]='P4' (len=P1) ** ** The string value P4 of length P1 (bytes) is stored in register P2. +** +** If P5!=0 and the content of register P3 is greater than zero, then +** the datatype of the register P2 is converted to BLOB. The content is +** the same sequence of bytes, it is merely interpreted as a BLOB instead +** of a string, as if it had been CAST. */ -case OP_String: { /* out2-prerelease */ +case OP_String: { /* out2 */ assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Str|MEM_Static|MEM_Term; pOut->z = pOp->p4.z; pOut->n = pOp->p1; pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); + if( pOp->p5 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); + pIn3 = &aMem[pOp->p3]; + assert( pIn3->flags & MEM_Int ); + if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; + } break; } @@ -70114,9 +74000,10 @@ case OP_String: { /* out2-prerelease */ ** NULL values will not compare equal even if SQLITE_NULLEQ is set on ** OP_Ne or OP_Eq. */ -case OP_Null: { /* out2-prerelease */ +case OP_Null: { /* out2 */ int cnt; u16 nullFlag; + pOut = out2Prerelease(p, pOp); cnt = pOp->p3-pOp->p2; assert( pOp->p3<=(p->nMem-p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; @@ -70151,8 +74038,9 @@ case OP_SoftNull: { ** P4 points to a blob of data P1 bytes long. Store this ** blob in register P2. */ -case OP_Blob: { /* out2-prerelease */ +case OP_Blob: { /* out2 */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); + pOut = out2Prerelease(p, pOp); sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); @@ -70167,7 +74055,7 @@ case OP_Blob: { /* out2-prerelease */ ** If the parameter is named, then its name appears in P4. ** The P4 value is used by sqlite3_bind_parameter_name(). */ -case OP_Variable: { /* out2-prerelease */ +case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); @@ -70176,6 +74064,7 @@ case OP_Variable: { /* out2-prerelease */ if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } + pOut = out2Prerelease(p, pOp); sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); UPDATE_MAX_BLOBSIZE(pOut); break; @@ -70210,10 +74099,11 @@ case OP_Move: { memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ - pOut->pScopyFrom += p1 - pOp->p2; + if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrompScopyFrom += pOp->p2 - p1; } #endif + Deephemeralize(pOut); REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; @@ -70352,7 +74242,7 @@ case OP_ResultRow: { /* Return SQLITE_ROW */ - p->pc = pc + 1; + p->pc = (int)(pOp - aOp) + 1; rc = SQLITE_ROW; goto vdbe_return; } @@ -70545,7 +74435,7 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ ** ** The interface used by the implementation of the aforementioned functions ** to retrieve the collation sequence set by this opcode is not available -** publicly, only to user functions defined in func.c. +** publicly. Only built-in functions have access to this feature. */ case OP_CollSeq: { assert( pOp->p4type==P4_COLLSEQ ); @@ -70555,10 +74445,10 @@ case OP_CollSeq: { break; } -/* Opcode: Function P1 P2 P3 P4 P5 +/* Opcode: Function0 P1 P2 P3 P4 P5 ** Synopsis: r[P3]=func(r[P2@P5]) ** -** Invoke a user function (P4 is a pointer to a Function structure that +** Invoke a user function (P4 is a pointer to a FuncDef object that ** defines the function) with P5 arguments taken from register P2 and ** successors. The result of the function is stored in register P3. ** Register P3 must not be one of the function inputs. @@ -70570,59 +74460,100 @@ case OP_CollSeq: { ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** See also: AggStep and AggFinal +** See also: Function, AggStep, AggFinal */ -case OP_Function: { - int i; - Mem *pArg; - sqlite3_context ctx; - sqlite3_value **apVal; +/* Opcode: Function P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with P5 arguments taken +** from register P2 and successors. The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** SQL functions are initially coded as OP_Function0 with P4 pointing +** to a FuncDef object. But on first evaluation, the P4 operand is +** automatically converted into an sqlite3_context object and the operation +** changed to this OP_Function opcode. In this way, the initialization of +** the sqlite3_context object occurs only once, rather than once for each +** evaluation of the function. +** +** See also: Function0, AggStep, AggFinal +*/ +case OP_Function0: { int n; + sqlite3_context *pCtx; + assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - apVal = p->apArg; - assert( apVal || n==0 ); assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - ctx.pOut = &aMem[pOp->p3]; - memAboutToChange(p, ctx.pOut); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pArg = &aMem[pOp->p2]; - for(i=0; ip2+i, pArg); + pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pOut = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_Function; + /* Fall through into OP_Function */ +} +case OP_Function: { + int i; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pOut = pOut; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - assert( pOp->p4type==P4_FUNCDEF ); - ctx.pFunc = pOp->p4.pFunc; - ctx.iOp = pc; - ctx.pVdbe = p; - MemSetTypeFlag(ctx.pOut, MEM_Null); - ctx.fErrorOrAux = 0; + memAboutToChange(p, pCtx->pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pCtx->pOut, MEM_Null); + pCtx->fErrorOrAux = 0; db->lastRowid = lastRowid; - (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ - if( ctx.fErrorOrAux ){ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(ctx.pOut)); - rc = ctx.isError; + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + rc = pCtx->isError; } - sqlite3VdbeDeleteAuxData(p, pc, pOp->p1); + sqlite3VdbeDeleteAuxData(p, pCtx->iOp, pOp->p1); } /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(ctx.pOut, encoding); - if( sqlite3VdbeMemTooBig(ctx.pOut) ){ - goto too_big; + if( pOut->flags & (MEM_Str|MEM_Blob) ){ + sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); + if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; } - REGISTER_TRACE(pOp->p3, ctx.pOut); - UPDATE_MAX_BLOBSIZE(ctx.pOut); + REGISTER_TRACE(pOp->p3, pCtx->pOut); + UPDATE_MAX_BLOBSIZE(pCtx->pOut); break; } @@ -70741,8 +74672,7 @@ case OP_MustBeInt: { /* jump, in1 */ rc = SQLITE_MISMATCH; goto abort_due_to_error; }else{ - pc = pOp->p2 - 1; - break; + goto jump_to_p2; } } } @@ -70786,9 +74716,9 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_Cast: { /* in1 */ - assert( pOp->p2>=SQLITE_AFF_NONE && pOp->p2<=SQLITE_AFF_REAL ); + assert( pOp->p2>=SQLITE_AFF_BLOB && pOp->p2<=SQLITE_AFF_REAL ); testcase( pOp->p2==SQLITE_AFF_TEXT ); - testcase( pOp->p2==SQLITE_AFF_NONE ); + testcase( pOp->p2==SQLITE_AFF_BLOB ); testcase( pOp->p2==SQLITE_AFF_NUMERIC ); testcase( pOp->p2==SQLITE_AFF_INTEGER ); testcase( pOp->p2==SQLITE_AFF_REAL ); @@ -70928,7 +74858,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else{ VdbeBranchTaken(2,3); if( pOp->p5 & SQLITE_JUMPIFNULL ){ - pc = pOp->p2-1; + goto jump_to_p2; } } break; @@ -70948,11 +74878,15 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn1, encoding, 1); + testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); + flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); } if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn3, encoding, 1); + testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); + flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); } } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); @@ -70976,6 +74910,12 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ default: res = res>=0; break; } + /* Undo any changes made by applyAffinity() to the input registers. */ + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; + assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); + pIn3->flags = flags3; + if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); @@ -70985,12 +74925,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else{ VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); if( res ){ - pc = pOp->p2-1; + goto jump_to_p2; } } - /* Undo any changes made by applyAffinity() to the input registers. */ - pIn1->flags = flags1; - pIn3->flags = flags3; break; } @@ -71085,11 +75022,11 @@ case OP_Compare: { */ case OP_Jump: { /* jump */ if( iCompare<0 ){ - pc = pOp->p1 - 1; VdbeBranchTaken(0,3); + VdbeBranchTaken(0,3); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ - pc = pOp->p2 - 1; VdbeBranchTaken(1,3); + VdbeBranchTaken(1,3); pOp = &aOp[pOp->p2 - 1]; }else{ - pc = pOp->p3 - 1; VdbeBranchTaken(2,3); + VdbeBranchTaken(2,3); pOp = &aOp[pOp->p3 - 1]; } break; } @@ -71199,7 +75136,7 @@ case OP_Once: { /* jump */ assert( pOp->p1nOnceFlag ); VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); if( p->aOnceFlag[pOp->p1] ){ - pc = pOp->p2-1; + goto jump_to_p2; }else{ p->aOnceFlag[pOp->p1] = 1; } @@ -71234,7 +75171,7 @@ case OP_IfNot: { /* jump, in1 */ } VdbeBranchTaken(c!=0, 2); if( c ){ - pc = pOp->p2-1; + goto jump_to_p2; } break; } @@ -71248,7 +75185,7 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; VdbeBranchTaken( (pIn1->flags & MEM_Null)!=0, 2); if( (pIn1->flags & MEM_Null)!=0 ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } @@ -71262,7 +75199,7 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2); if( (pIn1->flags & MEM_Null)==0 ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } @@ -71476,7 +75413,7 @@ case OP_Column: { } } - /* If after trying to extra new entries from the header, nHdrParsed is + /* If after trying to extract new entries from the header, nHdrParsed is ** still not up to p2, that means that the record has fewer than p2 ** columns. So the result will be either the default value or a NULL. */ @@ -71592,7 +75529,7 @@ case OP_Affinity: { ** The mapping from character to affinity is given by the SQLITE_AFF_ ** macros defined in sqliteInt.h. ** -** If P4 is NULL then all index fields have the affinity NONE. +** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { u8 *zNewRecord; /* A buffer to hold the data for the new record */ @@ -71600,7 +75537,7 @@ case OP_MakeRecord: { u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ i64 nByte; /* Data space required for this record */ - int nZero; /* Number of zero bytes at the end of the record */ + i64 nZero; /* Number of zero bytes at the end of the record */ int nVarint; /* Number of bytes in a varint */ u32 serial_type; /* Type field */ Mem *pData0; /* First field to be combined into the record */ @@ -71664,7 +75601,7 @@ case OP_MakeRecord: { len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ - sqlite3VdbeMemExpandBlob(pRec); + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; }else{ nZero += pRec->u.nZero; len -= pRec->u.nZero; @@ -71676,7 +75613,10 @@ case OP_MakeRecord: { nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); }while( (--pRec)>=pData0 ); - /* Add the initial header varint and total the size */ + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ testcase( nHdr==126 ); testcase( nHdr==127 ); if( nHdr<=126 ){ @@ -71689,7 +75629,7 @@ case OP_MakeRecord: { if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } @@ -71710,7 +75650,11 @@ case OP_MakeRecord: { pRec = pData0; do{ serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); assert( i==nHdr ); @@ -71736,7 +75680,7 @@ case OP_MakeRecord: { ** opened by cursor P1 in register P2 */ #ifndef SQLITE_OMIT_BTREECOUNT -case OP_Count: { /* out2-prerelease */ +case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; @@ -71744,6 +75688,7 @@ case OP_Count: { /* out2-prerelease */ assert( pCrsr ); nEntry = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3BtreeCount(pCrsr, &nEntry); + pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; break; } @@ -71782,8 +75727,7 @@ case OP_Savepoint: { /* A new savepoint cannot be created if there are active write ** statements (i.e. open read/write incremental blob handles). */ - sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " - "SQL statements in progress"); + sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); rc = SQLITE_BUSY; }else{ nName = sqlite3Strlen30(zName); @@ -71834,15 +75778,14 @@ case OP_Savepoint: { iSavepoint++; } if( !pSavepoint ){ - sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); + sqlite3VdbeError(p, "no such savepoint: %s", zName); rc = SQLITE_ERROR; }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are ** active write statements. */ - sqlite3SetString(&p->zErrMsg, db, - "cannot release savepoint - SQL statements in progress" - ); + sqlite3VdbeError(p, "cannot release savepoint - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else{ @@ -71857,7 +75800,7 @@ case OP_Savepoint: { } db->autoCommit = 1; if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; + p->pc = (int)(pOp - aOp); db->autoCommit = 0; p->rc = rc = SQLITE_BUSY; goto vdbe_return; @@ -71916,7 +75859,7 @@ case OP_Savepoint: { db->nDeferredImmCons = pSavepoint->nDeferredImmCons; } - if( !isTransaction ){ + if( !isTransaction || p1==SAVEPOINT_ROLLBACK ){ rc = sqlite3VtabSavepoint(db, p1, iSavepoint); if( rc!=SQLITE_OK ) goto abort_due_to_error; } @@ -71948,23 +75891,12 @@ case OP_AutoCommit: { assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); -#if 0 - if( turnOnAC && iRollback && db->nVdbeActive>1 ){ - /* If this instruction implements a ROLLBACK and other VMs are - ** still running, and a transaction is active, return an error indicating - ** that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else -#endif if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ - sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " - "SQL statements in progress"); + sqlite3VdbeError(p, "cannot commit transaction - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ @@ -71975,12 +75907,12 @@ case OP_AutoCommit: { goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -71991,7 +75923,7 @@ case OP_AutoCommit: { } goto vdbe_return; }else{ - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, (!desiredAutoCommit)?"cannot start a transaction within a transaction":( (iRollback)?"cannot rollback - no transaction is active": "cannot commit - no transaction is active")); @@ -72052,9 +75984,11 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ - p->pc = pc; - p->rc = rc = SQLITE_BUSY; + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); + if( (rc&0xff)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + p->rc = rc; goto vdbe_return; } if( rc!=SQLITE_OK ){ @@ -72083,7 +76017,12 @@ case OP_Transaction: { p->nStmtDefImmCons = db->nDeferredImmCons; } - /* Gather the schema version number for checking */ + /* Gather the schema version number for checking: + ** IMPLEMENTATION-OF: R-32195-19465 The schema version is used by SQLite + ** each time a query is executed to ensure that the internal cache of the + ** schema used when compiling the SQL query matches the schema of the + ** database against which the compiled query is actually executed. + */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); iGen = db->aDb[pOp->p1].pSchema->iGeneration; }else{ @@ -72127,7 +76066,7 @@ case OP_Transaction: { ** must be started or there must be an open cursor) before ** executing this instruction. */ -case OP_ReadCookie: { /* out2-prerelease */ +case OP_ReadCookie: { /* out2 */ int iMeta; int iDb; int iCookie; @@ -72141,6 +76080,7 @@ case OP_ReadCookie: { /* out2-prerelease */ assert( DbMaskTest(p->btreeMask, iDb) ); sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); + pOut = out2Prerelease(p, pOp); pOut->u.i = iMeta; break; } @@ -72251,31 +76191,29 @@ case OP_SetCookie: { /* in3 */ ** See also OpenRead. */ case OP_ReopenIdx: { + int nField; + KeyInfo *pKeyInfo; + int p2; + int iDb; + int wrFlag; + Btree *pX; VdbeCursor *pCur; + Db *pDb; - assert( pOp->p5==0 ); + assert( pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( pOp->p4type==P4_KEYINFO ); pCur = p->apCsr[pOp->p1]; if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ - break; + goto open_cursor_set_hints; } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -} case OP_OpenRead: -case OP_OpenWrite: { - int nField; - KeyInfo *pKeyInfo; - int p2; - int iDb; - int wrFlag; - Btree *pX; - VdbeCursor *pCur; - Db *pDb; +case OP_OpenWrite: - assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); - assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); + assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR|OPFLAG_SEEKEQ))==pOp->p5 ); + assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx || p->readOnly==0 ); @@ -72338,14 +76276,17 @@ case OP_OpenWrite: { pCur->pgnoRoot = p2; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; - assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); - sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); - /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has ** since moved into the btree layer. */ pCur->isTable = pOp->p4type!=P4_KEYINFO; + +open_cursor_set_hints: + assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); + assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); + sqlite3BtreeCursorHints(pCur->pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); break; } @@ -72461,7 +76402,7 @@ case OP_SequenceTest: { pC = p->apCsr[pOp->p1]; assert( pC->pSorter ); if( (pC->seqCount++)==0 ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } @@ -72508,6 +76449,26 @@ case OP_Close: { break; } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK +/* Opcode: ColumnsUsed P1 * * P4 * +** +** This opcode (which only exists if SQLite was compiled with +** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the +** table or index for cursor P1 are used. P4 is a 64-bit integer +** (P4_INT64) in which the first 63 bits are one for each of the +** first 63 columns of the table or index that are actually used +** by the cursor. The high-order bit is set if any column after +** the 64th is used. +*/ +case OP_ColumnsUsed: { + VdbeCursor *pC; + pC = p->apCsr[pOp->p1]; + assert( pC->pCursor ); + pC->maskUsed = *(u64*)pOp->p4.pI64; + break; +} +#endif + /* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -72606,6 +76567,22 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif + + /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and + ** OP_SeekLE opcodes are allowed, and these must be immediately followed + ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + */ +#ifdef SQLITE_DEBUG + if( sqlite3BtreeCursorHasHint(pC->pCursor, BTREE_SEEK_EQ) ){ + assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp[1].p1==pOp[0].p1 ); + assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p3==pOp[0].p3 ); + assert( pOp[1].p4.i==pOp[0].p4.i ); + } +#endif + if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -72622,7 +76599,7 @@ case OP_SeekGT: { /* jump, in3 */ if( (pIn3->flags & MEM_Real)==0 ){ /* If the P3 value cannot be converted into any kind of a number, ** then the seek is not possible, so jump to P2 */ - pc = pOp->p2 - 1; VdbeBranchTaken(1,2); + VdbeBranchTaken(1,2); goto jump_to_p2; break; } @@ -72713,7 +76690,7 @@ case OP_SeekGT: { /* jump, in3 */ assert( pOp->p2>0 ); VdbeBranchTaken(res!=0,2); if( res ){ - pc = pOp->p2 - 1; + goto jump_to_p2; } break; } @@ -72807,6 +76784,7 @@ case OP_NoConflict: /* jump, in3 */ case OP_NotFound: /* jump, in3 */ case OP_Found: { /* jump, in3 */ int alreadyExists; + int takeJump; int ii; VdbeCursor *pC; int res; @@ -72829,7 +76807,7 @@ case OP_Found: { /* jump, in3 */ pIn3 = &aMem[pOp->p3]; assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); - pFree = 0; /* Not needed. Only used to suppress a compiler warning. */ + pFree = 0; if( pOp->p4.i>0 ){ r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; @@ -72845,28 +76823,27 @@ case OP_Found: { /* jump, in3 */ }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; + takeJump = 0; if( pOp->opcode==OP_NoConflict ){ /* For the OP_NoConflict opcode, take the jump if any of the ** input fields are NULL, since any key with a NULL will not ** conflict */ - for(ii=0; iip2 - 1; VdbeBranchTaken(1,2); + for(ii=0; iinField; ii++){ + if( pIdxKey->aMem[ii].flags & MEM_Null ){ + takeJump = 1; break; } } } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); - if( pOp->p4.i==0 ){ - sqlite3DbFree(db, pFree); - } + sqlite3DbFree(db, pFree); if( rc!=SQLITE_OK ){ break; } @@ -72877,10 +76854,10 @@ case OP_Found: { /* jump, in3 */ pC->cacheStatus = CACHE_STALE; if( pOp->opcode==OP_Found ){ VdbeBranchTaken(alreadyExists!=0,2); - if( alreadyExists ) pc = pOp->p2 - 1; + if( alreadyExists ) goto jump_to_p2; }else{ - VdbeBranchTaken(alreadyExists==0,2); - if( !alreadyExists ) pc = pOp->p2 - 1; + VdbeBranchTaken(takeJump||alreadyExists==0,2); + if( takeJump || !alreadyExists ) goto jump_to_p2; } break; } @@ -72890,9 +76867,10 @@ case OP_Found: { /* jump, in3 */ ** ** P1 is the index of a cursor open on an SQL table btree (with integer ** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. If P1 does contain a record -** with rowid P3 then leave the cursor pointing at that record and fall -** through to the next instruction. +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. ** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). @@ -72924,15 +76902,21 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); + pC->seekResult = res; if( res!=0 ){ - pc = pOp->p2 - 1; + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + goto jump_to_p2; + } } - pC->seekResult = res; break; } @@ -72944,9 +76928,10 @@ case OP_NotExists: { /* jump, in3 */ ** The sequence number on the cursor is incremented after this ** instruction. */ -case OP_Sequence: { /* out2-prerelease */ +case OP_Sequence: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( p->apCsr[pOp->p1]!=0 ); + pOut = out2Prerelease(p, pOp); pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; } @@ -72967,7 +76952,7 @@ case OP_Sequence: { /* out2-prerelease */ ** generated record number. This P3 mechanism is used to help implement the ** AUTOINCREMENT feature. */ -case OP_NewRowid: { /* out2-prerelease */ +case OP_NewRowid: { /* out2 */ i64 v; /* The new rowid */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ int res; /* Result of an sqlite3BtreeLast() */ @@ -72977,12 +76962,12 @@ case OP_NewRowid: { /* out2-prerelease */ v = 0; res = 0; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( NEVER(pC->pCursor==0) ){ - /* The zero initialization above is all that is needed */ - }else{ + assert( pC->pCursor!=0 ); + { /* The next rowid or record number (different terms for the same ** thing) is obtained in a two-step algorithm. ** @@ -73197,14 +77182,15 @@ case OP_InsertInt: { break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Delete P1 P2 * P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. ** -** The cursor will be left pointing at either the next or the previous -** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within a Next loop. +** If the P5 parameter is non-zero, the cursor will be left pointing at +** either the next or the previous record in the table. If it is left +** pointing at the next record, then the next Next instruction will be a +** no-op. As a result, in this case it is OK to delete a record from within a +** Next loop. If P5 is zero, then the cursor is left in an undefined state. ** ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is ** incremented (otherwise not). @@ -73219,6 +77205,7 @@ case OP_InsertInt: { */ case OP_Delete: { VdbeCursor *pC; + u8 hasUpdateCallback; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -73226,22 +77213,27 @@ case OP_Delete: { assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ assert( pC->deferredMoveto==0 ); + hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable; + if( pOp->p5 && hasUpdateCallback ){ + sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget); + } + #ifdef SQLITE_DEBUG /* The seek operation that positioned the cursor prior to OP_Delete will ** have also set the pC->movetoTarget field to the rowid of the row that ** is being deleted */ - if( pOp->p4.z && pC->isTable ){ + if( pOp->p4.z && pC->isTable && pOp->p5==0 ){ i64 iKey = 0; sqlite3BtreeKeySize(pC->pCursor, &iKey); assert( pC->movetoTarget==iKey ); } #endif - rc = sqlite3BtreeDelete(pC->pCursor); + rc = sqlite3BtreeDelete(pC->pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + if( rc==SQLITE_OK && hasUpdateCallback ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); assert( pC->iDb>=0 ); @@ -73290,9 +77282,7 @@ case OP_SorterCompare: { res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2-1; - } + if( res ) goto jump_to_p2; break; }; @@ -73421,12 +77411,13 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2-prerelease */ +case OP_Rowid: { /* out2 */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; const sqlite3_module *pModule; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -73479,7 +77470,7 @@ case OP_NullRow: { break; } -/* Opcode: Last P1 P2 * * * +/* Opcode: Last P1 P2 P3 * * ** ** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. @@ -73506,12 +77497,13 @@ case OP_Last: { /* jump */ pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; + pC->seekResult = pOp->p3; #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; #endif if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); - if( res ) pc = pOp->p2 - 1; + if( res ) goto jump_to_p2; } break; } @@ -73542,9 +77534,9 @@ case OP_Sort: { /* jump */ ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -73575,9 +77567,7 @@ case OP_Rewind: { /* jump */ pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2 - 1; - } + if( res ) goto jump_to_p2; break; } @@ -73688,11 +77678,11 @@ case OP_Next: /* jump */ VdbeBranchTaken(res==0,2); if( res==0 ){ pC->nullRow = 0; - pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; #ifdef SQLITE_TEST sqlite3_search_count++; #endif + goto jump_to_p2_and_check_for_interrupt; }else{ pC->nullRow = 1; } @@ -73723,7 +77713,6 @@ case OP_Next: /* jump */ case OP_SorterInsert: /* in2 */ case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; - BtCursor *pCrsr; int nKey; const char *zKey; @@ -73733,18 +77722,17 @@ case OP_IdxInsert: { /* in2 */ assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); - pCrsr = pC->pCursor; if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pCrsr!=0 ); + assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ - if( isSorter(pC) ){ + if( pOp->opcode==OP_SorterInsert ){ rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; - rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, + rc = sqlite3BtreeInsert(pC->pCursor, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) ); assert( pC->deferredMoveto==0 ); @@ -73784,7 +77772,7 @@ case OP_IdxDelete: { #endif rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); if( rc==SQLITE_OK && res==0 ){ - rc = sqlite3BtreeDelete(pCrsr); + rc = sqlite3BtreeDelete(pCrsr, 0); } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; @@ -73800,11 +77788,12 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_IdxRowid: { /* out2-prerelease */ +case OP_IdxRowid: { /* out2 */ BtCursor *pCrsr; VdbeCursor *pC; i64 rowid; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -73917,9 +77906,7 @@ case OP_IdxGE: { /* jump */ res++; } VdbeBranchTaken(res>0,2); - if( res>0 ){ - pc = pOp->p2 - 1 ; - } + if( res>0 ) goto jump_to_p2; break; } @@ -73943,32 +77930,18 @@ case OP_IdxGE: { /* jump */ ** ** See also: Clear */ -case OP_Destroy: { /* out2-prerelease */ +case OP_Destroy: { /* out2 */ int iMoved; - int iCnt; - Vdbe *pVdbe; int iDb; assert( p->readOnly==0 ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - iCnt = 0; - for(pVdbe=db->pVdbe; pVdbe; pVdbe = pVdbe->pNext){ - if( pVdbe->magic==VDBE_MAGIC_RUN && pVdbe->bIsReader - && pVdbe->inVtabMethod<2 && pVdbe->pc>=0 - ){ - iCnt++; - } - } -#else - iCnt = db->nVdbeRead; -#endif + pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; - if( iCnt>1 ){ + if( db->nVdbeRead > db->nVDestroy+1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; }else{ iDb = pOp->p3; - assert( iCnt==1 ); assert( DbMaskTest(p->btreeMask, iDb) ); iMoved = 0; /* Not needed. Only to silence a warning. */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); @@ -74071,12 +78044,13 @@ case OP_ResetSorter: { ** ** See documentation on OP_CreateTable for additional information. */ -case OP_CreateIndex: /* out2-prerelease */ -case OP_CreateTable: { /* out2-prerelease */ +case OP_CreateIndex: /* out2 */ +case OP_CreateTable: { /* out2 */ int pgno; int flags; Db *pDb; + pOut = out2Prerelease(p, pOp); pgno = 0; assert( pOp->p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); @@ -74302,12 +78276,12 @@ case OP_RowSetRead: { /* jump, in1, out3 */ ){ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); - pc = pOp->p2 - 1; VdbeBranchTaken(1,2); + goto jump_to_p2_and_check_for_interrupt; }else{ /* A value was pulled from the index */ - sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); VdbeBranchTaken(0,2); + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); } goto check_for_interrupt; } @@ -74358,10 +78332,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ if( iSet ){ exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); - if( exists ){ - pc = pOp->p2 - 1; - break; - } + if( exists ) goto jump_to_p2; } if( iSet>=0 ){ sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); @@ -74420,7 +78391,7 @@ case OP_Program: { /* jump */ if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); + sqlite3VdbeError(p, "too many levels of trigger recursion"); break; } @@ -74450,7 +78421,7 @@ case OP_Program: { /* jump */ pFrame->v = p; pFrame->nChildMem = nMem; pFrame->nChildCsr = pProgram->nCsr; - pFrame->pc = pc; + pFrame->pc = (int)(pOp - aOp); pFrame->aMem = p->aMem; pFrame->nMem = p->nMem; pFrame->apCsr = p->apCsr; @@ -74460,6 +78431,9 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pFrame->anExec = p->anExec; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -74470,13 +78444,14 @@ case OP_Program: { /* jump */ pFrame = pRt->u.pFrame; assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); assert( pProgram->nCsr==pFrame->nChildCsr ); - assert( pc==pFrame->pc ); + assert( (int)(pOp - aOp)==pFrame->pc ); } p->nFrame++; pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; @@ -74487,7 +78462,10 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; - pc = -1; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = 0; +#endif + pOp = &aOp[-1]; memset(p->aOnceFlag, 0, p->nOnceFlag); break; @@ -74505,9 +78483,10 @@ case OP_Program: { /* jump */ ** the value of the P1 argument to the value of the P1 argument to the ** calling OP_Program instruction. */ -case OP_Param: { /* out2-prerelease */ +case OP_Param: { /* out2 */ VdbeFrame *pFrame; Mem *pIn; + pOut = out2Prerelease(p, pOp); pFrame = p->pFrame; pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem); @@ -74551,10 +78530,10 @@ case OP_FkCounter: { case OP_FkIfZero: { /* jump */ if( pOp->p1 ){ VdbeBranchTaken(db->nDeferredCons==0 && db->nDeferredImmCons==0, 2); - if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; }else{ VdbeBranchTaken(p->nFkConstraint==0 && db->nDeferredImmCons==0, 2); - if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; + if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; } break; } @@ -74591,109 +78570,189 @@ case OP_MemMax: { /* in2 */ } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 * * * -** Synopsis: if r[P1]>0 goto P2 +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 ** -** If the value of register P1 is 1 or greater, jump to P2. +** Register P1 must contain an integer. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. ** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. +** If the initial value of register P1 is less than 1, then the +** value is unchanged and control passes through to the next instruction. */ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); if( pIn1->u.i>0 ){ - pc = pOp->p2 - 1; + pIn1->u.i -= pOp->p3; + goto jump_to_p2; + } + break; +} + +/* Opcode: SetIfNotPos P1 P2 P3 * * +** Synopsis: if r[P1]<=0 then r[P2]=P3 +** +** Register P1 must contain an integer. +** If the value of register P1 is not positive (if it is less than 1) then +** set the value of register P2 to be the integer P3. +*/ +case OP_SetIfNotPos: { /* in1, in2 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + if( pIn1->u.i<=0 ){ + pOut = out2Prerelease(p, pOp); + pOut->u.i = pOp->p3; } break; } -/* Opcode: IfNeg P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +/* Opcode: IfNotZero P1 P2 P3 * * +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 ** -** Register P1 must contain an integer. Add literal P3 to the value in -** register P1 then if the value of register P1 is less than zero, jump to P2. +** Register P1 must contain an integer. If the content of register P1 is +** initially nonzero, then subtract P3 from the value in register P1 and +** jump to P2. If register P1 is initially zero, leave it unchanged +** and fall through. */ -case OP_IfNeg: { /* jump, in1 */ +case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; VdbeBranchTaken(pIn1->u.i<0, 2); - if( pIn1->u.i<0 ){ - pc = pOp->p2 - 1; + if( pIn1->u.i ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; } break; } -/* Opcode: IfZero P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]==0 goto P2 +/* Opcode: DecrJumpZero P1 P2 * * * +** Synopsis: if (--r[P1])==0 goto P2 ** -** The register P1 must contain an integer. Add literal P3 to the -** value in register P1. If the result is exactly 0, jump to P2. +** Register P1 must hold an integer. Decrement the value in register P1 +** then jump to P2 if the new value is exactly zero. */ -case OP_IfZero: { /* jump, in1 */ +case OP_DecrJumpZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; + pIn1->u.i--; VdbeBranchTaken(pIn1->u.i==0, 2); - if( pIn1->u.i==0 ){ - pc = pOp->p2 - 1; - } + if( pIn1->u.i==0 ) goto jump_to_p2; break; } -/* Opcode: AggStep * P2 P3 P4 P5 + +/* Opcode: JumpZeroIncr P1 P2 * * * +** Synopsis: if (r[P1]++)==0 ) goto P2 +** +** The register P1 must contain an integer. If register P1 is initially +** zero, then jump to P2. Increment register P1 regardless of whether or +** not the jump is taken. +*/ +case OP_JumpZeroIncr: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken(pIn1->u.i==0, 2); + if( (pIn1->u.i++)==0 ) goto jump_to_p2; + break; +} + +/* Opcode: AggStep0 * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** ** Execute the step function for an aggregate. The ** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Use register -** P3 as the accumulator. +** structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -case OP_AggStep: { +/* Opcode: AggStep * P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the step function for an aggregate. The +** function has P5 arguments. P4 is a pointer to an sqlite3_context +** object that is used to run the function. Register P3 is +** as the accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +** +** This opcode is initially coded as OP_AggStep0. On first evaluation, +** the FuncDef stored in P4 is converted into an sqlite3_context and +** the opcode is changed. In this way, the initialization of the +** sqlite3_context only happens once, instead of on each call to the +** step function. +*/ +case OP_AggStep0: { int n; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCDEF ); + n = pOp->p5; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pMem = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_AggStep; + /* Fall through into OP_AggStep */ +} +case OP_AggStep: { int i; + sqlite3_context *pCtx; Mem *pMem; - Mem *pRec; Mem t; - sqlite3_context ctx; - sqlite3_value **apVal; - n = pOp->p5; - assert( n>=0 ); - pRec = &aMem[pOp->p2]; - apVal = p->apArg; - assert( apVal || n==0 ); - for(i=0; ip4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + pMem = &aMem[pOp->p3]; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + if( pCtx->pMem != pMem ){ + pCtx->pMem = pMem; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - ctx.pFunc = pOp->p4.pFunc; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - ctx.pMem = pMem = &aMem[pOp->p3]; + +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + pMem->n++; sqlite3VdbeMemInit(&t, db, MEM_Null); - ctx.pOut = &t; - ctx.isError = 0; - ctx.pVdbe = p; - ctx.iOp = pc; - ctx.skipFlag = 0; - (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); - rc = ctx.isError; + pCtx->pOut = &t; + pCtx->fErrorOrAux = 0; + pCtx->skipFlag = 0; + (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); + rc = pCtx->isError; + } + sqlite3VdbeMemRelease(&t); + }else{ + assert( t.flags==MEM_Null ); } - if( ctx.skipFlag ){ + if( pCtx->skipFlag ){ assert( pOp[-1].opcode==OP_CollSeq ); i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } - sqlite3VdbeMemRelease(&t); break; } @@ -74717,7 +78776,7 @@ case OP_AggFinal: { assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem)); + sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); @@ -74731,8 +78790,8 @@ case OP_AggFinal: { /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint @@ -74750,6 +78809,7 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ @@ -74775,7 +78835,7 @@ case OP_Checkpoint: { ** ** Write a string containing the final journal-mode to register P2. */ -case OP_JournalMode: { /* out2-prerelease */ +case OP_JournalMode: { /* out2 */ Btree *pBt; /* Btree to change journal mode of */ Pager *pPager; /* Pager associated with pBt */ int eNew; /* New journal mode */ @@ -74784,6 +78844,7 @@ case OP_JournalMode: { /* out2-prerelease */ const char *zFilename; /* Name of database file for pPager */ #endif + pOut = out2Prerelease(p, pOp); eNew = pOp->p3; assert( eNew==PAGER_JOURNALMODE_DELETE || eNew==PAGER_JOURNALMODE_TRUNCATE @@ -74820,7 +78881,7 @@ case OP_JournalMode: { /* out2-prerelease */ ){ if( !db->autoCommit || db->nVdbeRead>1 ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, "cannot change %s wal mode from within a transaction", (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); @@ -74859,7 +78920,6 @@ case OP_JournalMode: { /* out2-prerelease */ } eNew = sqlite3PagerSetJournalMode(pPager, eNew); - pOut = &aMem[pOp->p2]; pOut->flags = MEM_Str|MEM_Static|MEM_Term; pOut->z = (char *)sqlite3JournalModename(eNew); pOut->n = sqlite3Strlen30(pOut->z); @@ -74900,8 +78960,8 @@ case OP_IncrVacuum: { /* jump */ rc = sqlite3BtreeIncrVacuum(pBt); VdbeBranchTaken(rc==SQLITE_DONE,2); if( rc==SQLITE_DONE ){ - pc = pOp->p2 - 1; rc = SQLITE_OK; + goto jump_to_p2; } break; } @@ -74952,7 +79012,7 @@ case OP_TableLock: { rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); if( (rc&0xFF)==SQLITE_LOCKED ){ const char *z = pOp->p4.z; - sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + sqlite3VdbeError(p, "database table is locked: %s", z); } } break; @@ -74979,13 +79039,29 @@ case OP_VBegin: { #endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VCreate P1 * * P4 * +/* Opcode: VCreate P1 P2 * * * ** -** P4 is the name of a virtual table in database P1. Call the xCreate method -** for that table. +** P2 is a register that holds the name of a virtual table in database +** P1. Call the xCreate method for that table. */ case OP_VCreate: { - rc = sqlite3VtabCallCreate(db, pOp->p1, pOp->p4.z, &p->zErrMsg); + Mem sMem; /* For storing the record being decoded */ + const char *zTab; /* Name of the virtual table */ + + memset(&sMem, 0, sizeof(sMem)); + sMem.db = db; + /* Because P2 is always a static string, it is impossible for the + ** sqlite3VdbeMemCopy() to fail */ + assert( (aMem[pOp->p2].flags & MEM_Str)!=0 ); + assert( (aMem[pOp->p2].flags & MEM_Static)!=0 ); + rc = sqlite3VdbeMemCopy(&sMem, &aMem[pOp->p2]); + assert( rc==SQLITE_OK ); + zTab = (const char*)sqlite3_value_text(&sMem); + assert( zTab || db->mallocFailed ); + if( zTab ){ + rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg); + } + sqlite3VdbeMemRelease(&sMem); break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -74997,9 +79073,9 @@ case OP_VCreate: { ** of that table. */ case OP_VDestroy: { - p->inVtabMethod = 2; + db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); - p->inVtabMethod = 0; + db->nVDestroy--; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -75015,14 +79091,17 @@ case OP_VOpen: { VdbeCursor *pCur; sqlite3_vtab_cursor *pVtabCursor; sqlite3_vtab *pVtab; - sqlite3_module *pModule; + const sqlite3_module *pModule; assert( p->bIsReader ); pCur = 0; pVtabCursor = 0; pVtab = pOp->p4.pVtab->pVtab; - pModule = (sqlite3_module *)pVtab->pModule; - assert(pVtab && pModule); + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + break; + } + pModule = pVtab->pModule; rc = pModule->xOpen(pVtab, &pVtabCursor); sqlite3VtabImportErrmsg(p, pVtab); if( SQLITE_OK==rc ){ @@ -75033,9 +79112,11 @@ case OP_VOpen: { pCur = allocateCursor(p, pOp->p1, 0, -1, 0); if( pCur ){ pCur->pVtabCursor = pVtabCursor; + pVtab->nRef++; }else{ - db->mallocFailed = 1; + assert( db->mallocFailed ); pModule->xClose(pVtabCursor); + goto no_mem; } } break; @@ -75091,27 +79172,19 @@ case OP_VFilter: { /* jump */ iQuery = (int)pQuery->u.i; /* Invoke the xFilter method */ - { - res = 0; - apArg = p->apArg; - for(i = 0; iinVtabMethod = 1; - rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); - p->inVtabMethod = 0; - sqlite3VtabImportErrmsg(p, pVtab); - if( rc==SQLITE_OK ){ - res = pModule->xEof(pVtabCursor); - } - VdbeBranchTaken(res!=0,2); - if( res ){ - pc = pOp->p2 - 1; - } + res = 0; + apArg = p->apArg; + for(i = 0; ixFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc==SQLITE_OK ){ + res = pModule->xEof(pVtabCursor); } pCur->nullRow = 0; - + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -75190,9 +79263,7 @@ case OP_VNext: { /* jump */ ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ - p->inVtabMethod = 1; rc = pModule->xNext(pCur->pVtabCursor); - p->inVtabMethod = 0; sqlite3VtabImportErrmsg(p, pVtab); if( rc==SQLITE_OK ){ res = pModule->xEof(pCur->pVtabCursor); @@ -75200,7 +79271,7 @@ case OP_VNext: { /* jump */ VdbeBranchTaken(!res,2); if( !res ){ /* If there is data, jump to P2 */ - pc = pOp->p2 - 1; + goto jump_to_p2_and_check_for_interrupt; } goto check_for_interrupt; } @@ -75267,7 +79338,7 @@ case OP_VRename: { */ case OP_VUpdate: { sqlite3_vtab *pVtab; - sqlite3_module *pModule; + const sqlite3_module *pModule; int nArg; int i; sqlite_int64 rowid; @@ -75279,7 +79350,11 @@ case OP_VUpdate: { ); assert( p->readOnly==0 ); pVtab = pOp->p4.pVtab->pVtab; - pModule = (sqlite3_module *)pVtab->pModule; + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + break; + } + pModule = pVtab->pModule; nArg = pOp->p2; assert( pOp->p4type==P4_VTAB ); if( ALWAYS(pModule->xUpdate) ){ @@ -75319,7 +79394,8 @@ case OP_VUpdate: { ** ** Write the current number of pages in database P1 to memory cell P2. */ -case OP_Pagecount: { /* out2-prerelease */ +case OP_Pagecount: { /* out2 */ + pOut = out2Prerelease(p, pOp); pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); break; } @@ -75335,10 +79411,11 @@ case OP_Pagecount: { /* out2-prerelease */ ** ** Store the maximum page count after the change in register P2. */ -case OP_MaxPgcnt: { /* out2-prerelease */ +case OP_MaxPgcnt: { /* out2 */ unsigned int newMax; Btree *pBt; + pOut = out2Prerelease(p, pOp); pBt = db->aDb[pOp->p1].pBt; newMax = 0; if( pOp->p3 ){ @@ -75367,9 +79444,6 @@ case OP_Init: { /* jump */ char *zTrace; char *z; - if( pOp->p2 ){ - pc = pOp->p2 - 1; - } #ifndef SQLITE_OMIT_TRACE if( db->xTrace && !p->doingRerun @@ -75397,6 +79471,7 @@ case OP_Init: { /* jump */ } #endif /* SQLITE_DEBUG */ #endif /* SQLITE_OMIT_TRACE */ + if( pOp->p2 ) goto jump_to_p2; break; } @@ -75428,8 +79503,8 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { u64 endTime = sqlite3Hwtime(); - if( endTime>start ) pOp->cycles += endTime - start; - pOp->cnt++; + if( endTime>start ) pOrigOp->cycles += endTime - start; + pOrigOp->cnt++; } #endif @@ -75439,16 +79514,16 @@ default: { /* This is really OP_Noop and OP_Explain */ ** the evaluator loop. So we can leave it out when NDEBUG is defined. */ #ifndef NDEBUG - assert( pc>=-1 && pcnOp ); + assert( pOp>=&aOp[-1] && pOp<&aOp[p->nOp-1] ); #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ if( rc!=0 ) printf("rc=%d\n",rc); - if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ - registerTrace(pOp->p2, &aMem[pOp->p2]); + if( pOrigOp->opflags & (OPFLG_OUT2) ){ + registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]); } - if( pOp->opflags & OPFLG_OUT3 ){ - registerTrace(pOp->p3, &aMem[pOp->p3]); + if( pOrigOp->opflags & OPFLG_OUT3 ){ + registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); } } #endif /* SQLITE_DEBUG */ @@ -75463,7 +79538,7 @@ default: { /* This is really OP_Noop and OP_Explain */ p->rc = rc; testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", - pc, p->zSql, p->zErrMsg); + (int)(pOp - aOp), p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; rc = SQLITE_ERROR; @@ -75485,7 +79560,7 @@ default: { /* This is really OP_Noop and OP_Explain */ ** is encountered. */ too_big: - sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); + sqlite3VdbeError(p, "string or blob too big"); rc = SQLITE_TOOBIG; goto vdbe_error_halt; @@ -75493,7 +79568,7 @@ default: { /* This is really OP_Noop and OP_Explain */ */ no_mem: db->mallocFailed = 1; - sqlite3SetString(&p->zErrMsg, db, "out of memory"); + sqlite3VdbeError(p, "out of memory"); rc = SQLITE_NOMEM; goto vdbe_error_halt; @@ -75504,7 +79579,7 @@ default: { /* This is really OP_Noop and OP_Explain */ assert( p->zErrMsg==0 ); if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc!=SQLITE_IOERR_NOMEM ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); } goto vdbe_error_halt; @@ -75515,7 +79590,7 @@ default: { /* This is really OP_Noop and OP_Explain */ assert( db->u1.isInterrupted ); rc = SQLITE_INTERRUPT; p->rc = rc; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); goto vdbe_error_halt; } @@ -75537,6 +79612,8 @@ default: { /* This is really OP_Noop and OP_Explain */ ** This file contains code used to implement incremental BLOB I/O. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_INCRBLOB @@ -75626,7 +79703,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ /* ** Open a blob handle. */ -SQLITE_API int sqlite3_blob_open( +SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( sqlite3* db, /* The database connection */ const char *zDb, /* The attached database containing the blob */ const char *zTable, /* The table containing the blob */ @@ -75675,8 +79752,18 @@ SQLITE_API int sqlite3_blob_open( Parse *pParse = 0; Incrblob *pBlob = 0; - flags = !!flags; /* flags = (flags ? 1 : 0); */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppBlob==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif *ppBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + flags = !!flags; /* flags = (flags ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); @@ -75759,7 +79846,8 @@ SQLITE_API int sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ - if( pIdx->aiColumn[j]==iCol ){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ zFault = "indexed"; } } @@ -75853,7 +79941,7 @@ SQLITE_API int sqlite3_blob_open( ** Close a blob handle that was previously created using ** sqlite3_blob_open(). */ -SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; int rc; sqlite3 *db; @@ -75890,10 +79978,9 @@ static int blobReadWrite( sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; - if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ + if( n<0 || iOffset<0 || ((sqlite3_int64)iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -75911,10 +79998,10 @@ static int blobReadWrite( sqlite3VdbeFinalize(v); p->pStmt = 0; }else{ - db->errCode = rc; v->rc = rc; } } + sqlite3Error(db, rc); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -75923,14 +80010,14 @@ static int blobReadWrite( /* ** Read data from a blob handle. */ -SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); } /* ** Write data to a blob handle. */ -SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); } @@ -75940,7 +80027,7 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; return (p && p->pStmt) ? p->nByte : 0; } @@ -75955,7 +80042,7 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ ** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) ** immediately return SQLITE_ABORT. */ -SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ int rc; Incrblob *p = (Incrblob *)pBlob; sqlite3 *db; @@ -76091,7 +80178,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** The sorter is running in multi-threaded mode if (a) the library was built ** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater ** than zero, and (b) worker threads have been enabled at runtime by calling -** sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, ...). +** "PRAGMA threads=N" with some value of N greater than 0. ** ** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted @@ -76126,6 +80213,8 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** thread to merge the output of each of the others to a single PMA for ** the main thread to read from. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various @@ -76136,6 +80225,13 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ # define SQLITE_DEBUG_SORTER_THREADS 1 #endif +/* +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_PMASZ (1<<29) + /* ** Private objects used by the sorter */ @@ -76273,6 +80369,7 @@ struct MergeEngine { ** after the thread has finished are not dire. So we don't worry about ** memory barriers and such here. */ +typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ @@ -76280,10 +80377,12 @@ struct SortSubtask { UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ int nPMA; /* Number of PMAs currently in file */ + SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ }; + /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. @@ -76310,9 +80409,13 @@ struct VdbeSorter { u8 bUseThreads; /* True to use background threads */ u8 iPrev; /* Previous thread used to flush PMA */ u8 nTask; /* Size of aTask[] array */ + u8 typeMask; SortSubtask aTask[1]; /* One or more subtasks */ }; +#define SORTER_TYPE_INTEGER 0x01 +#define SORTER_TYPE_TEXT 0x02 + /* ** An instance of the following object is used to read records out of a ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. @@ -76430,9 +80533,6 @@ struct SorterRecord { */ #define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) -/* The minimum PMA size is set to this value multiplied by the database -** page size in bytes. */ -#define SORTER_MIN_WORKING 10 /* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 @@ -76727,32 +80827,162 @@ static int vdbePmaReaderInit( return rc; } +/* +** A version of vdbeSorterCompare() that assumes that it has already been +** determined that the first field of key1 is equal to the first field of +** key2. +*/ +static int vdbeSorterCompareTail( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + UnpackedRecord *r2 = pTask->pUnpacked; + if( *pbKey2Cached==0 ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); +} /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, ** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** -** Before returning, object (pTask->pUnpacked) is populated with the -** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it -** is assumed that the (pTask->pUnpacked) structure already contains the -** unpacked key to use as key2. +** If IN/OUT parameter *pbKey2Cached is true when this function is called, +** it is assumed that (pTask->pUnpacked) contains the unpacked version +** of key2. If it is false, (pTask->pUnpacked) is populated with the unpacked +** version of key2 and *pbKey2Cached set to true before returning. ** ** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ UnpackedRecord *r2 = pTask->pUnpacked; - if( pKey2 ){ + if( !*pbKey2Cached ){ sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is a TEXT value and that the collation +** sequence to compare them with is BINARY. +*/ +static int vdbeSorterCompareText( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + + int n1; + int n2; + int res; + + getVarint32(&p1[1], n1); n1 = (n1 - 13) / 2; + getVarint32(&p2[1], n2); n2 = (n2 - 13) / 2; + res = memcmp(v1, v2, MIN(n1, n2)); + if( res==0 ){ + res = n1 - n2; + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else{ + if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + } + + return res; +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is an INTEGER value. +*/ +static int vdbeSorterCompareInt( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const int s1 = p1[1]; /* Left hand serial type */ + const int s2 = p2[1]; /* Right hand serial type */ + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + int res; /* Return value */ + + assert( (s1>0 && s1<7) || s1==8 || s1==9 ); + assert( (s2>0 && s2<7) || s2==8 || s2==9 ); + + if( s1>7 && s2>7 ){ + res = s1 - s2; + }else{ + if( s1==s2 ){ + if( (*v1 ^ *v2) & 0x80 ){ + /* The two values have different signs */ + res = (*v1 & 0x80) ? -1 : +1; + }else{ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8 }; + int i; + res = 0; + for(i=0; i7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; + }else{ + res = s1 - s2; + } + assert( res!=0 ); + + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else{ + if( *v2 & 0x80 ) res = +1; + } + } + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + + return res; +} + /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** @@ -76820,9 +81050,13 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; - if( nField && nWorker==0 ) pKeyInfo->nField = nField; + if( nField && nWorker==0 ){ + pKeyInfo->nXField += (pKeyInfo->nField - nField); + pKeyInfo->nField = nField; + } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; + pSorter->iPrev = nWorker-1; pSorter->bUseThreads = (pSorter->nTask>1); pSorter->db = db; for(i=0; inTask; i++){ @@ -76831,16 +81065,15 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } if( !sqlite3TempInMemory(db) ){ - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + u32 szPma = sqlite3GlobalConfig.szPma; + pSorter->mnPmaSize = szPma * pgsz; mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; - - /* If the application has not configure scratch memory using - ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory - ** allocations. If scratch memory has been configured, then assume - ** large memory allocations should be avoided to prevent heap - ** fragmentation. + if( mxCache<(int)szPma ) mxCache = (int)szPma; + pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_PMASZ); + + /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of + ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary + ** large heap allocations. */ if( sqlite3GlobalConfig.pScratch==0 ){ assert( pSorter->iMemory==0 ); @@ -76849,6 +81082,12 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; } } + + if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + ){ + pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; + } } return rc; @@ -76873,30 +81112,24 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ */ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); - pTask->pUnpacked = 0; #if SQLITE_MAX_WORKER_THREADS>0 /* pTask->list.aMemory can only be non-zero if it was handed memory ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ if( pTask->list.aMemory ){ sqlite3_free(pTask->list.aMemory); - pTask->list.aMemory = 0; }else #endif { assert( pTask->list.aMemory==0 ); vdbeSorterRecordFree(0, pTask->list.pList); } - pTask->list.pList = 0; if( pTask->file.pFd ){ sqlite3OsCloseFree(pTask->file.pFd); - pTask->file.pFd = 0; - pTask->file.iEof = 0; } if( pTask->file2.pFd ){ sqlite3OsCloseFree(pTask->file2.pFd); - pTask->file2.pFd = 0; - pTask->file2.iEof = 0; } + memset(pTask, 0, sizeof(SortSubtask)); } #ifdef SQLITE_DEBUG_SORTER_THREADS @@ -77076,6 +81309,7 @@ SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); + pTask->pSorter = pSorter; } if( pSorter->list.aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->list.pList); @@ -77114,12 +81348,12 @@ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ */ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ - int rc = sqlite3OsTruncate(pFd, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFd, 0, (int)nByte, &p); - sqlite3OsUnfetch(pFd, 0, p); - } + void *p = 0; + int chunksize = 4*1024; + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); } } #else @@ -77137,6 +81371,7 @@ static int vdbeSorterOpenTempFile( sqlite3_file **ppFd ){ int rc; + if( sqlite3FaultSim(202) ) return SQLITE_IOERR_ACCESS; rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | @@ -77184,28 +81419,42 @@ static void vdbeSorterMerge( ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? SRVAL(p2) : 0; + int bCached = 0; while( p1 && p2 ){ int res; - res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); + res = pTask->xCompare( + pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal + ); + if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; p1 = p1->u.pNext; - pVal2 = 0; }else{ *pp = p2; - pp = &p2->u.pNext; + pp = &p2->u.pNext; p2 = p2->u.pNext; - if( p2==0 ) break; - pVal2 = SRVAL(p2); + bCached = 0; } } *pp = p1 ? p1 : p2; *ppOut = pFinal; } +/* +** Return the SorterCompare function to compare values collected by the +** sorter object passed as the only argument. +*/ +static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ + if( p->typeMask==SORTER_TYPE_INTEGER ){ + return vdbeSorterCompareInt; + }else if( p->typeMask==SORTER_TYPE_TEXT ){ + return vdbeSorterCompareText; + } + return vdbeSorterCompare; +} + /* ** Sort the linked list of records headed at pTask->pList. Return ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if @@ -77220,12 +81469,14 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ rc = vdbeSortAllocUnpacked(pTask); if( rc!=SQLITE_OK ) return rc; + p = pList->pList; + pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pList->pList; while( p ){ SorterRecord *pNext; if( pList->aMemory ){ @@ -77439,13 +81690,12 @@ static int vdbeMergeEngineStep( int i; /* Index of aTree[] to recalculate */ PmaReader *pReadr1; /* First PmaReader to compare */ PmaReader *pReadr2; /* Second PmaReader to compare */ - u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ + int bCached = 0; /* Find the first two PmaReaders to compare. The one that was just ** advanced (iPrev) and the one next to it in the array. */ pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; - pKey2 = pReadr2->aKey; for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ @@ -77455,8 +81705,8 @@ static int vdbeMergeEngineStep( }else if( pReadr2->pFd==0 ){ iRes = -1; }else{ - iRes = vdbeSorterCompare(pTask, - pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey + iRes = pTask->xCompare(pTask, &bCached, + pReadr1->aKey, pReadr1->nKey, pReadr2->aKey, pReadr2->nKey ); } @@ -77478,9 +81728,9 @@ static int vdbeMergeEngineStep( if( iRes<0 || (iRes==0 && pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; - pKey2 = pReadr2->aKey; + bCached = 0; }else{ - if( pReadr1->pFd ) pKey2 = 0; + if( pReadr1->pFd ) bCached = 0; pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } @@ -77587,6 +81837,16 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( int bFlush; /* True to flush contents of memory to PMA */ int nReq; /* Bytes of memory required */ int nPMA; /* Bytes of PMA space required */ + int t; /* serial type of first record field */ + + getVarint32((const u8*)&pVal->z[1], t); + if( t>0 && t<10 && t!=7 ){ + pSorter->typeMask &= SORTER_TYPE_INTEGER; + }else if( t>10 && (t & 0x01) ){ + pSorter->typeMask &= SORTER_TYPE_TEXT; + }else{ + pSorter->typeMask = 0; + } assert( pSorter ); @@ -77852,10 +82112,12 @@ static void vdbeMergeEngineCompare( }else if( p2->pFd==0 ){ iRes = i1; }else{ + SortSubtask *pTask = pMerger->pTask; + int bCached = 0; int res; - assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ - res = vdbeSorterCompare( - pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + assert( pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = pTask->xCompare( + pTask, &bCached, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -77879,11 +82141,12 @@ static void vdbeMergeEngineCompare( #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -/* Forward reference. -** The vdbeIncrMergeInit() and vdbePmaReaderIncrMergeInit() routines call each -** other (when building a merge tree). +/* +** Forward reference required as the vdbeIncrMergeInit() and +** vdbePmaReaderIncrInit() routines are called mutually recursively when +** building a merge tree. */ -static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* ** Initialize the MergeEngine object passed as the second argument. Once this @@ -77930,7 +82193,7 @@ static int vdbeMergeEngineInit( ** better advantage of multi-processor hardware. */ rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrMergeInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } if( rc!=SQLITE_OK ) return rc; } @@ -77942,17 +82205,15 @@ static int vdbeMergeEngineInit( } /* -** Initialize the IncrMerge field of a PmaReader. -** -** If the PmaReader passed as the first argument is not an incremental-reader -** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves -** to open and/or initialize the temp file related fields of the IncrMerge +** The PmaReader passed as the first argument is guaranteed to be an +** incremental-reader (pReadr->pIncr!=0). This function serves to open +** and/or initialize the temp file related fields of the IncrMerge ** object at (pReadr->pIncr). ** ** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders -** in the sub-tree headed by pReadr are also initialized. Data is then loaded -** into the buffers belonging to pReadr and it is set to -** point to the first key in its range. +** in the sub-tree headed by pReadr are also initialized. Data is then +** loaded into the buffers belonging to pReadr and it is set to point to +** the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed ** to be a multi-threaded PmaReader and this function is being called in a @@ -77979,59 +82240,62 @@ static int vdbeMergeEngineInit( static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; + SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; /* eMode is always INCRINIT_NORMAL in single-threaded mode */ assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); - if( pIncr ){ - SortSubtask *pTask = pIncr->pTask; - sqlite3 *db = pTask->pSorter->db; - - rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr. A multi-theaded IncrMerge object - ** requires two temp files to itself, whereas a single-threaded object - ** only requires a region of pTask->file2. */ - if( rc==SQLITE_OK ){ - int mxSz = pIncr->mxSz; + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ + if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; #if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->bUseThread ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); - } - }else + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else #endif - /*if( !pIncr->bUseThread )*/{ - if( pTask->file2.pFd==0 ){ - assert( pTask->file2.iEof>0 ); - rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); - pTask->file2.iEof = 0; - } - if( rc==SQLITE_OK ){ - pIncr->aFile[1].pFd = pTask->file2.pFd; - pIncr->iStartOff = pTask->file2.iEof; - pTask->file2.iEof += mxSz; - } + /*if( !pIncr->bUseThread )*/{ + if( pTask->file2.pFd==0 ){ + assert( pTask->file2.iEof>0 ); + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += mxSz; } } + } #if SQLITE_MAX_WORKER_THREADS>0 - if( rc==SQLITE_OK && pIncr->bUseThread ){ - /* Use the current thread to populate aFile[1], even though this - ** PmaReader is multi-threaded. The reason being that this function - ** is already running in background thread pIncr->pTask->thread. */ - assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); - rc = vdbeIncrPopulate(pIncr); - } + if( rc==SQLITE_OK && pIncr->bUseThread ){ + /* Use the current thread to populate aFile[1], even though this + ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object, + ** then this function is already running in background thread + ** pIncr->pTask->thread. + ** + ** If this is the INCRINIT_ROOT object, then it is running in the + ** main VDBE thread. But that is Ok, as that thread cannot return + ** control to the VDBE or proceed with anything useful until the + ** first results are ready from this merger object anyway. + */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); + rc = vdbeIncrPopulate(pIncr); + } #endif - if( rc==SQLITE_OK - && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) - ){ - rc = vdbePmaReaderNext(pReadr); - } + if( rc==SQLITE_OK && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) ){ + rc = vdbePmaReaderNext(pReadr); } + return rc; } @@ -78040,7 +82304,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ ** The main routine for vdbePmaReaderIncrMergeInit() operations run in ** background threads. */ -static void *vdbePmaReaderBgInit(void *pCtx){ +static void *vdbePmaReaderBgIncrInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) @@ -78048,20 +82312,36 @@ static void *vdbePmaReaderBgInit(void *pCtx){ pReader->pIncr->pTask->bDone = 1; return pRet; } +#endif /* -** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) -** on the PmaReader object passed as the first argument. -** -** This call will initialize the various fields of the pReadr->pIncr -** structure and, if it is a multi-threaded IncrMerger, launch a -** background thread to populate aFile[1]. +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes +** the vdbePmaReaderIncrMergeInit() function with the parameters passed to +** this routine to initialize the incremental merge. +** +** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), +** then a background thread is launched to call vdbePmaReaderIncrMergeInit(). +** Or, if the IncrMerger is single threaded, the same function is called +** using the current thread. */ -static int vdbePmaReaderBgIncrInit(PmaReader *pReadr){ - void *pCtx = (void*)pReadr; - return vdbeSorterCreateThread(pReadr->pIncr->pTask, vdbePmaReaderBgInit, pCtx); -} +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ + IncrMerger *pIncr = pReadr->pIncr; /* Incremental merger */ + int rc = SQLITE_OK; /* Return code */ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + assert( pIncr->bUseThread==0 || eMode==INCRINIT_TASK ); + if( pIncr->bUseThread ){ + void *pCtx = (void*)pReadr; + rc = vdbeSorterCreateThread(pIncr->pTask, vdbePmaReaderBgIncrInit, pCtx); + }else #endif + { + rc = vdbePmaReaderIncrMergeInit(pReadr, eMode); + } + } + return rc; +} /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 @@ -78273,6 +82553,11 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ MergeEngine *pMain = 0; #if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; + int i; + SorterCompare xCompare = vdbeSorterGetCompare(pSorter); + for(i=0; inTask; i++){ + pSorter->aTask[i].xCompare = xCompare; + } #endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); @@ -78301,15 +82586,21 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ } } for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + /* Check that: + ** + ** a) The incremental merge object is configured to use the + ** right task, and + ** b) If it is using task (nTask-1), it is configured to run + ** in single-threaded mode. This is important, as the + ** root merge (INCRINIT_ROOT) will be using the same task + ** object. + */ PmaReader *p = &pMain->aReadr[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ - if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrMergeInit(p, INCRINIT_TASK); - }else{ - rc = vdbePmaReaderBgIncrInit(p); - } - } + assert( p->pIncr==0 || ( + (p->pIncr->pTask==&pSorter->aTask[iTask]) /* a */ + && (iTask!=pSorter->nTask-1 || p->pIncr->bUseThread==0) /* b */ + )); + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); } } pMain = 0; @@ -78400,6 +82691,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, in }else #endif /*if( !pSorter->bUseThreads )*/ { + assert( pSorter->pMerger!=0 ); assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); } @@ -78539,6 +82831,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( ** 2) The sqlite3JournalCreate() function is called. */ #ifdef SQLITE_ENABLE_ATOMIC_WRITE +/* #include "sqliteInt.h" */ /* @@ -78786,6 +83079,7 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. */ +/* #include "sqliteInt.h" */ /* Forward references to internal structures */ typedef struct MemJournal MemJournal; @@ -79041,6 +83335,7 @@ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ ** This file contains routines used for walking the parser tree for ** an SQL statement. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -79133,6 +83428,11 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; @@ -79199,6 +83499,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** resolve all identifiers by associating them with a particular ** table and column. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -79212,7 +83513,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -79220,7 +83521,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } @@ -79229,30 +83530,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -79263,7 +83540,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; ** ** The nSubquery parameter specifies how many levels of subquery the -** alias is removed from the original expression. The usually value is +** alias is removed from the original expression. The usual value is ** zero but it might be more if the alias is contained within a subquery ** of the original expression. The Expr.op2 field of TK_AGG_FUNCTION ** structures must be increased by the nSubquery amount. @@ -79283,23 +83560,14 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); - assert( pOrig->flags & EP_Resolved ); db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -79431,9 +83699,10 @@ static int lookupName( testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IsCheck ); if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ - /* Silently ignore database qualifiers inside CHECK constraints and partial - ** indices. Do not raise errors because that might break legacy and - ** because it does not hurt anything to just ignore the database name. */ + /* Silently ignore database qualifiers inside CHECK constraints and + ** partial indices. Do not raise errors because that might break + ** legacy and because it does not hurt anything to just ignore the + ** database name. */ zDb = 0; }else{ for(i=0; inDb; i++){ @@ -79490,7 +83759,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->jointype & JT_NATURAL ) continue; + if( pItem->fg.jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -79504,6 +83773,11 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + /* RIGHT JOIN not (yet) supported */ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ @@ -79537,7 +83811,7 @@ static int lookupName( break; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ /* IMP: R-51414-32910 */ /* IMP: R-44911-55124 */ iCol = -1; @@ -79566,8 +83840,13 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) - && HasRowid(pMatch->pTab) ){ + if( cnt==0 + && cntTab==1 + && pMatch + && (pNC->ncFlags & NC_IdxExpr)==0 + && sqlite3IsRowid(zCol) + && VisibleRowid(pMatch->pTab) + ){ cnt = 1; pExpr->iColumn = -1; /* IMP: R-44911-55124 */ pExpr->affinity = SQLITE_AFF_INTEGER; @@ -79586,9 +83865,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** or HAVING clauses, or as part of a larger expression in the ORDER BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. TO DO: Issue a warning + ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 @@ -79685,7 +83964,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -79726,36 +84005,25 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask /* Set of contexts for which prohibited */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -79768,7 +84036,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -79821,7 +84089,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->affinity = SQLITE_AFF_INTEGER; break; } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + && !defined(SQLITE_OMIT_SUBQUERY) */ /* A lone identifier is the name of a column. */ @@ -79839,6 +84108,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -79868,7 +84139,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); + notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); @@ -79886,21 +84157,22 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ - sqlite3ErrorMsg(pParse, "second argument to likelihood() must be a " - "constant between 0.0 and 1.0"); + sqlite3ErrorMsg(pParse, + "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); pNC->nErr++; } }else{ - /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to - ** likelihood(X, 0.0625). - ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for - ** likelihood(X,0.0625). - ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for - ** likelihood(X,0.9375). - ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to - ** likelihood(X,0.9375). */ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is + ** equivalent to likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is + ** short-hand for likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand + ** for likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent + ** to likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } #ifndef SQLITE_OMIT_AUTHORIZATION @@ -79915,7 +84187,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ + ExprSetProperty(pExpr,EP_ConstFunc); + } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); @@ -79961,8 +84244,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -79972,8 +84254,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } } @@ -80167,9 +84448,11 @@ static int resolveCompoundOrderBy( if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ - assert( pItem->pExpr->op==TK_COLLATE ); - assert( pItem->pExpr->pLeft==pE ); - pItem->pExpr->pLeft = pNew; + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; } sqlite3ExprDelete(db, pE); pItem->u.x.iOrderByCol = (u16)iCol; @@ -80226,7 +84509,8 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); return 1; } - resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, zType,0); + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, + zType,0); } } return 0; @@ -80314,7 +84598,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -80359,6 +84642,20 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sqlite3ResolveExprNames(&sNC, p->pOffset) ){ return WRC_Abort; } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } /* Recursively resolve names in all subqueries */ @@ -80373,7 +84670,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->isCorrelated flag if this is the case. */ + ** pItem->fg.isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -80382,8 +84679,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->isCorrelated==0 && nRef<=0 ); - pItem->isCorrelated = (nRef!=0); + assert( pItem->fg.isCorrelated==0 && nRef<=0 ); + pItem->fg.isCorrelated = (nRef!=0); } } @@ -80395,14 +84692,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -80435,18 +84725,46 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg; + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } + /* Process the ORDER BY clause for singleton SELECT statements. ** The ORDER BY clause for compounds SELECT statements is handled ** below, after all of the result-sets for all of the elements of ** the compound have been resolved. + ** + ** If there is an ORDER BY clause on a term of a compound-select other + ** than the right-most term, then that is a syntax error. But the error + ** is not detected until much later, and so we need to go ahead and + ** resolve those symbols on the incorrect ORDER BY for consistency. */ - if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ + if( isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ + && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") + ){ return WRC_Abort; } if( db->mallocFailed ){ @@ -80471,6 +84789,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* If this is part of a compound SELECT, check that it has the right + ** number of expressions in the select list. */ + if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pNext); + return WRC_Abort; + } + /* Advance to the next term of the compound */ p = p->pPrior; @@ -80573,6 +84898,22 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return ExprHasProperty(pExpr, EP_Error); } +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + assert( pList!=0 ); + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + } + return WRC_Continue; +} /* ** Resolve all names in all expressions of a SELECT and in all @@ -80616,15 +84957,14 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( SQLITE_PRIVATE void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - int i; /* Loop counter */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; @@ -80635,13 +84975,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ - return; - } - } - } + if( pList ) sqlite3ResolveExprListNames(&sNC, pList); } /************** End of resolve.c *********************************************/ @@ -80660,6 +84994,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Return the 'affinity' of the expression pExpr if any. @@ -80716,10 +85051,11 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( Parse *pParse, /* Parsing context */ Expr *pExpr, /* Add the "COLLATE" clause to this expression */ - const Token *pCollName /* Name of collating sequence */ + const Token *pCollName, /* Name of collating sequence */ + int dequote /* True to dequote pCollName */ ){ if( pCollName->n>0 ){ - Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); if( pNew ){ pNew->pLeft = pExpr; pNew->flags |= EP_Collate|EP_Skip; @@ -80733,11 +85069,11 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, con assert( zC!=0 ); s.z = zC; s.n = sqlite3Strlen30(s.z); - return sqlite3ExprAddCollateToken(pParse, pExpr, &s); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -80748,7 +85084,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -80779,9 +85115,9 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); break; } - if( p->pTab!=0 - && (op==TK_AGG_COLUMN || op==TK_COLUMN + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER) + && p->pTab!=0 ){ /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally ** a TK_COLUMN but was previously evaluated and cached in a register */ @@ -80793,10 +85129,25 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ break; } if( p->flags & EP_Collate ){ - if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ p = p->pLeft; }else{ - p = p->pRight; + Expr *pNext = p->pRight; + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + /* p->flags holds EP_Collate and p->pLeft->flags does not. And + ** p->x.pSelect cannot. So if p->x.pLeft exists, it must hold at + ** least one EP_Collate. Thus the following two ALWAYS. */ + if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){ + int i; + for(i=0; ALWAYS(ix.pList->nExpr); i++){ + if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ + pNext = p->x.pList->a[i].pExpr; + break; + } + } + } + p = pNext; } }else{ break; @@ -80822,13 +85173,13 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ return SQLITE_AFF_NUMERIC; }else{ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; } }else if( !aff1 && !aff2 ){ /* Neither side of the comparison is a column. Compare the ** results directly. */ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ assert( aff1==0 || aff2==0 ); @@ -80852,7 +85203,7 @@ static char comparisonAffinity(Expr *pExpr){ }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); }else if( !aff ){ - aff = SQLITE_AFF_NONE; + aff = SQLITE_AFF_BLOB; } return aff; } @@ -80866,7 +85217,7 @@ static char comparisonAffinity(Expr *pExpr){ SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); switch( aff ){ - case SQLITE_AFF_NONE: + case SQLITE_AFF_BLOB: return 1; case SQLITE_AFF_TEXT: return idx_affinity==SQLITE_AFF_TEXT; @@ -81002,6 +85353,9 @@ static void heightOfSelect(Select *p, int *pnHeight){ ** Expr.pSelect member has a height of 1. Any other expression ** has a height equal to the maximum height of any other ** referenced Expr plus one. +** +** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, +** if appropriate. */ static void exprSetHeight(Expr *p){ int nHeight = 0; @@ -81009,8 +85363,9 @@ static void exprSetHeight(Expr *p){ heightOfExpr(p->pRight, &nHeight); if( ExprHasProperty(p, EP_xIsSelect) ){ heightOfSelect(p->x.pSelect, &nHeight); - }else{ + }else if( p->x.pList ){ heightOfExprList(p->x.pList, &nHeight); + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); } p->nHeight = nHeight + 1; } @@ -81019,8 +85374,12 @@ static void exprSetHeight(Expr *p){ ** Set the Expr.nHeight variable using the exprSetHeight() function. If ** the height is greater than the maximum allowed expression depth, ** leave an error in pParse. +** +** Also propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. */ -SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; exprSetHeight(p); sqlite3ExprCheckHeight(pParse, p->nHeight); } @@ -81034,8 +85393,17 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ heightOfSelect(p, &nHeight); return nHeight; } -#else - #define exprSetHeight(y) +#else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ +/* +** Propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } +} +#define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ /* @@ -81047,7 +85415,7 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote +** If dequote is false, no dequoting is performed. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. @@ -81137,11 +85505,11 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( }else{ if( pRight ){ pRoot->pRight = pRight; - pRoot->flags |= EP_Collate & pRight->flags; + pRoot->flags |= EP_Propagate & pRight->flags; } if( pLeft ){ pRoot->pLeft = pLeft; - pRoot->flags |= EP_Collate & pLeft->flags; + pRoot->flags |= EP_Propagate & pLeft->flags; } exprSetHeight(pRoot); } @@ -81162,7 +85530,7 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( const Token *pToken /* Argument token */ ){ Expr *p; - if( op==TK_AND && pLeft && pRight ){ + if( op==TK_AND && pLeft && pRight && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ @@ -81241,7 +85609,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * } pNew->x.pList = pList; assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - sqlite3ExprSetHeight(pParse, pNew); + sqlite3ExprSetHeightAndFlags(pParse, pNew); return pNew; } @@ -81648,16 +86016,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; + pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - pNewItem->isCorrelated = pOldItem->isCorrelated; - pNewItem->viaCoroutine = pOldItem->viaCoroutine; - pNewItem->isRecursive = pOldItem->isRecursive; - pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->pIBIndex = pOldItem->pIBIndex; + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; @@ -81772,6 +86142,20 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( return 0; } +/* +** Set the sort order for the last element on the given ExprList. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ + if( p==0 ) return; + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); + assert( p->nExpr>0 ); + if( iSortOrder<0 ){ + assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); + return; + } + p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; +} + /* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. @@ -81857,20 +86241,40 @@ SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.i to 0 if the expression is -** not constant. +** Return the bitwise-OR of all Expr.flags fields in the given +** ExprList. +*/ +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ + int i; + u32 m = 0; + if( pList ){ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( ALWAYS(pExpr) ) m |= pExpr->flags; + } + } + return m; +} + +/* +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() pWalker->u.i==1 -** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 -** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprIsTableConstant() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. ** ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing -** an existing schema and 3 when processing a new statement. A bound +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound ** parameter raises an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions @@ -81879,23 +86283,25 @@ SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 2 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -81904,18 +86310,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } case TK_VARIABLE: - if( pWalker->u.i==4 ){ + if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out ** of the sqlite_master table */ pExpr->op = TK_NULL; - }else if( pWalker->u.i==3 ){ + }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from ** sqlite3_prepare() causes an error */ - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ @@ -81927,21 +86337,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -81949,21 +86360,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2); + return exprIsConst(p, 2, 0); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -81973,7 +86394,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 3+isInit); + return exprIsConst(p, 4+isInit, 0); } /* @@ -82040,7 +86461,8 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( p->pTab!=0 ); - return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } @@ -82058,7 +86480,7 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; - if( aff==SQLITE_AFF_NONE ) return 1; + if( aff==SQLITE_AFF_BLOB ) return 1; while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } op = p->op; if( op==TK_REGISTER ) op = p->op2; @@ -82155,13 +86577,13 @@ SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int j1; + int addr1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } @@ -82268,7 +86690,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int ** ephemeral table. */ p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); - if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ + if( pParse->nErr==0 && isCandidateForInOpt(p) ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table . */ Expr *pExpr; /* Expression */ @@ -82509,7 +86931,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( int r1, r2, r3; if( !affinity ){ - affinity = SQLITE_AFF_NONE; + affinity = SQLITE_AFF_BLOB; } if( pKeyInfo ){ assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); @@ -82593,6 +87015,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[1]); pSel->iLimit = 0; + pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } @@ -82704,7 +87127,7 @@ static void sqlite3ExprCodeIN( } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeGoto(v, destIfFalse); } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); @@ -82722,7 +87145,7 @@ static void sqlite3ExprCodeIN( int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeGoto(v, destIfNull); sqlite3VdbeJumpHere(v, addr1); } } @@ -82760,7 +87183,7 @@ static void sqlite3ExprCodeIN( ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1; + int addr1; /* First check to see if the LHS is contained in the RHS. If so, ** then the answer is TRUE the presence of NULLs in the RHS does @@ -82768,12 +87191,12 @@ static void sqlite3ExprCodeIN( ** answer is NULL if the RHS contains NULLs and the answer is ** FALSE if the RHS is NULL-free. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeGoto(v, destIfFalse); + sqlite3VdbeJumpHere(v, addr1); } } } @@ -82783,17 +87206,6 @@ static void sqlite3ExprCodeIN( } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Duplicate an 8-byte value -*/ -static char *dup8bytes(Vdbe *v, const char *in){ - char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); - if( out ){ - memcpy(out, in, 8); - } - return out; -} - #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Generate an instruction that will put the floating point @@ -82806,12 +87218,10 @@ static char *dup8bytes(Vdbe *v, const char *in){ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - char *zV; sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); } } #endif @@ -82837,10 +87247,8 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ - char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); }else{ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); @@ -82881,7 +87289,8 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int int idxLru; struct yColCache *p; - assert( iReg>0 ); /* Register numbers are always positive */ + /* Unless an error has occurred, register numbers are always positive. */ + assert( iReg>0 || pParse->nErr || pParse->db->mallocFailed ); assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ /* The SQLITE_ColumnCache flag disables the column cache. This is used @@ -83005,6 +87414,28 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -83190,8 +87621,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iColumn + pParse->ckBase; break; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, @@ -83212,7 +87644,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); break; } case TK_NULL: { @@ -83251,10 +87683,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iTable; break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; - } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -83444,7 +87872,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + inReg = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); break; } @@ -83485,7 +87913,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ @@ -83514,7 +87942,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); if( nFarg && constMask==0 ){ @@ -83629,7 +88057,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ if( pExpr->iColumn>=0 && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -83706,7 +88137,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3VdbeGoto(v, endLabel); sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } @@ -83882,268 +88313,6 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ exprToRegister(pExpr, iMem); } -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable explanation of an expression tree. -*/ -SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ - const char *zBinOp = 0; /* Binary operator */ - const char *zUniOp = 0; /* Unary operator */ - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( pExpr==0 ){ - sqlite3TreeViewLine(pView, "nil"); - sqlite3TreeViewPop(pView); - return; - } - switch( pExpr->op ){ - case TK_AGG_COLUMN: { - sqlite3TreeViewLine(pView, "AGG{%d:%d}", - pExpr->iTable, pExpr->iColumn); - break; - } - case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); - }else{ - sqlite3TreeViewLine(pView, "{%d:%d}", - pExpr->iTable, pExpr->iColumn); - } - break; - } - case TK_INTEGER: { - if( pExpr->flags & EP_IntValue ){ - sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); - }else{ - sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); - } - break; - } -#ifndef SQLITE_OMIT_FLOATING_POINT - case TK_FLOAT: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_STRING: { - sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); - break; - } - case TK_NULL: { - sqlite3TreeViewLine(pView,"NULL"); - break; - } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_VARIABLE: { - sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); - break; - } - case TK_REGISTER: { - sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); - break; - } - case TK_AS: { - sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - case TK_ID: { - sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken); - break; - } -#ifndef SQLITE_OMIT_CAST - case TK_CAST: { - /* Expressions of the form: CAST(pLeft AS token) */ - sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } -#endif /* SQLITE_OMIT_CAST */ - case TK_LT: zBinOp = "LT"; break; - case TK_LE: zBinOp = "LE"; break; - case TK_GT: zBinOp = "GT"; break; - case TK_GE: zBinOp = "GE"; break; - case TK_NE: zBinOp = "NE"; break; - case TK_EQ: zBinOp = "EQ"; break; - case TK_IS: zBinOp = "IS"; break; - case TK_ISNOT: zBinOp = "ISNOT"; break; - case TK_AND: zBinOp = "AND"; break; - case TK_OR: zBinOp = "OR"; break; - case TK_PLUS: zBinOp = "ADD"; break; - case TK_STAR: zBinOp = "MUL"; break; - case TK_MINUS: zBinOp = "SUB"; break; - case TK_REM: zBinOp = "REM"; break; - case TK_BITAND: zBinOp = "BITAND"; break; - case TK_BITOR: zBinOp = "BITOR"; break; - case TK_SLASH: zBinOp = "DIV"; break; - case TK_LSHIFT: zBinOp = "LSHIFT"; break; - case TK_RSHIFT: zBinOp = "RSHIFT"; break; - case TK_CONCAT: zBinOp = "CONCAT"; break; - case TK_DOT: zBinOp = "DOT"; break; - - case TK_UMINUS: zUniOp = "UMINUS"; break; - case TK_UPLUS: zUniOp = "UPLUS"; break; - case TK_BITNOT: zUniOp = "BITNOT"; break; - case TK_NOT: zUniOp = "NOT"; break; - case TK_ISNULL: zUniOp = "ISNULL"; break; - case TK_NOTNULL: zUniOp = "NOTNULL"; break; - - case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - - case TK_AGG_FUNCTION: - case TK_FUNCTION: { - ExprList *pFarg; /* List of function arguments */ - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } - if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); - }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); - } - if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); - } - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_EXISTS: { - sqlite3TreeViewLine(pView, "EXISTS-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_SELECT: { - sqlite3TreeViewLine(pView, "SELECT-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_IN: { - sqlite3TreeViewLine(pView, "IN"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - }else{ - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - } - break; - } -#endif /* SQLITE_OMIT_SUBQUERY */ - - /* - ** x BETWEEN y AND z - ** - ** This is equivalent to - ** - ** x>=y AND x<=z - ** - ** X is stored in pExpr->pLeft. - ** Y is stored in pExpr->pList->a[0].pExpr. - ** Z is stored in pExpr->pList->a[1].pExpr. - */ - case TK_BETWEEN: { - Expr *pX = pExpr->pLeft; - Expr *pY = pExpr->x.pList->a[0].pExpr; - Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3TreeViewLine(pView, "BETWEEN"); - sqlite3TreeViewExpr(pView, pX, 1); - sqlite3TreeViewExpr(pView, pY, 1); - sqlite3TreeViewExpr(pView, pZ, 0); - break; - } - case TK_TRIGGER: { - /* If the opcode is TK_TRIGGER, then the expression is a reference - ** to a column in the new.* or old.* pseudo-tables available to - ** trigger programs. In this case Expr.iTable is set to 1 for the - ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn - ** is set to the column of the pseudo-table to read, or to -1 to - ** read the rowid field. - */ - sqlite3TreeViewLine(pView, "%s(%d)", - pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); - break; - } - case TK_CASE: { - sqlite3TreeViewLine(pView, "CASE"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - break; - } -#ifndef SQLITE_OMIT_TRIGGER - case TK_RAISE: { - const char *zType = "unk"; - switch( pExpr->affinity ){ - case OE_Rollback: zType = "rollback"; break; - case OE_Abort: zType = "abort"; break; - case OE_Fail: zType = "fail"; break; - case OE_Ignore: zType = "ignore"; break; - } - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); - break; - } -#endif - default: { - sqlite3TreeViewLine(pView, "op=%d", pExpr->op); - break; - } - } - if( zBinOp ){ - sqlite3TreeViewLine(pView, "%s", zBinOp); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExpr(pView, pExpr->pRight, 0); - }else if( zUniOp ){ - sqlite3TreeViewLine(pView, "%s", zUniOp); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable explanation of an expression list. -*/ -SQLITE_PRIVATE void sqlite3TreeViewExprList( - TreeView *pView, - const ExprList *pList, - u8 moreToFollow, - const char *zLabel -){ - int i; - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; - if( pList==0 ){ - sqlite3TreeViewLine(pView, "%s (empty)", zLabel); - }else{ - sqlite3TreeViewLine(pView, "%s", zLabel); - for(i=0; inExpr; i++){ - sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); -#if 0 - if( pList->a[i].zName ){ - sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); - } - if( pList->a[i].bSpanIsTab ){ - sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); - } -#endif - } - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. @@ -84160,11 +88329,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, n; + int i, j, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -84172,13 +88343,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; - Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg @@ -84355,14 +88527,14 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { if( exprAlwaysTrue(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ /* No-op */ }else{ @@ -84518,7 +88690,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int #endif default: { if( exprAlwaysFalse(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ /* no-op */ }else{ @@ -84535,6 +88707,21 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ReleaseTempReg(pParse, regFree2); } +/* +** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before +** code generation, and that copy is deleted after code generation. This +** ensures that the original pExpr is unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ + sqlite3 *db = pParse->db; + Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed==0 ){ + sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); + } + sqlite3ExprDelete(db, pCopy); +} + + /* ** Do a deep comparison of two expression trees. Return 0 if the two ** expressions are completely identical. Return 1 if they differ only @@ -84579,7 +88766,9 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ return 2; } if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } @@ -84589,7 +88778,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ if( sqlite3ExprCompare(pA->pLeft, pB->pLeft, iTab) ) return 2; if( sqlite3ExprCompare(pA->pRight, pB->pRight, iTab) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; - if( ALWAYS((combinedFlags & EP_Reduced)==0) ){ + if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){ if( pA->iColumn!=pB->iColumn ) return 2; if( pA->iTable!=pB->iTable && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; @@ -84691,10 +88880,11 @@ static int exprSrcCount(Walker *pWalker, Expr *pExpr){ int i; struct SrcCount *p = pWalker->u.pSrcCount; SrcList *pSrc = p->pSrc; - for(i=0; inSrc; i++){ + int nSrc = pSrc ? pSrc->nSrc : 0; + for(i=0; iiTable==pSrc->a[i].iCursor ) break; } - if( inSrc ){ + if( inThis++; }else{ p->nOther++; @@ -85007,6 +89197,7 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ ** This file contains C code routines that used to generate VDBE code ** that implements the ALTER TABLE command. */ +/* #include "sqliteInt.h" */ /* ** The code in this file only exists if we are not omitting the @@ -85120,6 +89311,7 @@ static void renameParentFunc( n = sqlite3GetToken(z, &token); }while( token==TK_SPACE ); + if( token==TK_ILLEGAL ) break; zParent = sqlite3DbStrNDup(db, (const char *)z, n); if( zParent==0 ) break; sqlite3Dequote(zParent); @@ -85484,7 +89676,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( #ifndef SQLITE_OMIT_VIRTUALTABLE if( pVTab ){ int i = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); sqlite3MayAbort(pParse); } @@ -85595,14 +89787,14 @@ SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minForm if( ALWAYS(v) ){ int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - int j1; + int addr1; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); - j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + addr1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); } @@ -85684,7 +89876,10 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ */ if( pDflt ){ sqlite3_value *pVal = 0; - if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ db->mallocFailed = 1; return; } @@ -85965,6 +90160,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE +/* #include "sqliteInt.h" */ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 @@ -86272,7 +90468,7 @@ static void statInit( p->mxSample = mxSample; p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; - p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; + p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); /* Set up the Stat4Accum.a[] and aBest[] arrays */ p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; @@ -86767,7 +90963,7 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut); sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } @@ -86838,7 +91034,7 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ @@ -86860,7 +91056,7 @@ static void analyzeOneTable( } /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + sqlite3VdbeLoadString(v, regIdxname, zIdxName); VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* @@ -86922,7 +91118,7 @@ static void analyzeOneTable( #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); @@ -86974,7 +91170,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + sqlite3VdbeGoto(v, endDistinctTest); /* @@ -87010,6 +91206,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -87018,7 +91215,7 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp); sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); @@ -87059,12 +91256,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif @@ -87281,23 +91476,28 @@ static void decodeIntArray( if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 - assert( pIndex!=0 ); + assert( pIndex!=0 ); { #else - if( pIndex ) + if( pIndex ){ #endif - while( z[0] ){ - if( sqlite3_strglob("unordered*", z)==0 ){ - pIndex->bUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); - } + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } #ifdef SQLITE_ENABLE_COSTMULT - else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ - pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); - } + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } #endif - while( z[0]!=0 && z[0]!=' ' ) z++; - while( z[0]==' ' ) z++; + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } } } @@ -87338,14 +91538,17 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ + tRowcnt *aiRowEst = 0; int nCol = pIndex->nKeyCol+1; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero( - sizeof(tRowcnt) * nCol - ); - if( aiRowEst==0 ) pInfo->db->mallocFailed = 1; -#else - tRowcnt * const aiRowEst = 0; + /* Index.aiRowEst may already be set here if there are duplicate + ** sqlite_stat1 entries for this index. In that case just clobber + ** the old data with the new instead of allocating a new array. */ + if( pIndex->aiRowEst==0 ){ + pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); + if( pIndex->aiRowEst==0 ) pInfo->db->mallocFailed = 1; + } + aiRowEst = pIndex->aiRowEst; #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); @@ -87415,7 +91618,7 @@ static void initAvgEq(Index *pIdx){ i64 nSum100 = 0; /* Number of terms contributing to sumEq */ i64 nDist100; /* Number of distinct values in index */ - if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){ + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ nRow = pFinal->anLt[iCol]; nDist100 = (i64)100 * pFinal->anDLt[iCol]; nSample--; @@ -87423,6 +91626,7 @@ static void initAvgEq(Index *pIdx){ nRow = pIdx->aiRowEst[0]; nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index. Set sumEq to the sum of @@ -87684,7 +91888,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); @@ -87721,6 +91925,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_ATTACH /* @@ -87859,6 +92064,7 @@ static void attachFunc( "attached databases must use the same text encoding as main database"); rc = SQLITE_ERROR; } + sqlite3BtreeEnter(aNew->pBt); pPager = sqlite3BtreePager(aNew->pBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3BtreeSecureDelete(aNew->pBt, @@ -87866,6 +92072,7 @@ static void attachFunc( #ifndef SQLITE_OMIT_PAGER_PRAGMAS sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); #endif + sqlite3BtreeLeave(aNew->pBt); } aNew->safety_level = 3; aNew->zName = sqlite3DbStrDup(db, zName); @@ -87898,7 +92105,7 @@ static void attachFunc( case SQLITE_NULL: /* No key specified. Use the key from the main database */ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey>0 || sqlite3BtreeGetReserve(db->aDb[0].pBt)>0 ){ + if( nKey>0 || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); } break; @@ -88005,7 +92212,7 @@ static void detachFunc( sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; - sqlite3ResetAllSchemasOfConnection(db); + sqlite3CollapseDatabaseArray(db); return; detach_error: @@ -88039,7 +92246,6 @@ static void codeAttach( SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) ){ - pParse->nErr++; goto attach_end; } @@ -88067,7 +92273,7 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); @@ -88309,6 +92515,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** systems that do not need this facility may omit it by recompiling ** the library with -DSQLITE_OMIT_AUTHORIZATION=1 */ +/* #include "sqliteInt.h" */ /* ** All of the code in this file may be omitted by defining a single @@ -88361,11 +92568,14 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** Setting the auth function to NULL disables this hook. The default ** setting of the auth function is NULL. */ -SQLITE_API int sqlite3_set_authorizer( +SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3 *db, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; @@ -88576,6 +92786,7 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** COMMIT ** ROLLBACK */ +/* #include "sqliteInt.h" */ /* ** This routine is called when a new SQL statement is beginning to @@ -88695,9 +92906,11 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( pParse->pToplevel==0 ); db = pParse->db; - if( db->mallocFailed ) return; if( pParse->nested ) return; - if( pParse->nErr ) return; + if( db->mallocFailed || pParse->nErr ){ + if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + return; + } /* Begin by generating some termination code at the end of the ** vdbe program @@ -88743,6 +92956,8 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ @@ -88772,14 +92987,14 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); + sqlite3VdbeGoto(v, 1); } } /* Get the VDBE program ready for execution */ - if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ + if( v && pParse->nErr==0 && !db->mallocFailed ){ assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ @@ -88860,7 +93075,7 @@ SQLITE_PRIVATE int sqlite3UserAuthTable(const char *zTable){ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - assert( zName!=0 ); + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); #if SQLITE_USER_AUTHENTICATION @@ -88907,6 +93122,17 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } + } +#endif if( zDbase ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ @@ -88914,7 +93140,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( } pParse->checkSchema = 1; } -#if SQLITE_USER_AUTHENICATION +#if SQLITE_USER_AUTHENTICATION else if( pParse->db->auth.authLevelpnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -89112,7 +93338,7 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -89179,13 +93405,11 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqliteDeleteColumnNames(db, pTable); + sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -89315,14 +93539,12 @@ SQLITE_PRIVATE int sqlite3TwoPartName( if( ALWAYS(pName2!=0) && pName2->n>0 ){ if( db->init.busy ) { sqlite3ErrorMsg(pParse, "corrupt database"); - pParse->nErr++; return -1; } *pUnqual = pName2; iDb = sqlite3FindDb(db, pName1); if( iDb<0 ){ sqlite3ErrorMsg(pParse, "unknown database %T", pName1); - pParse->nErr++; return -1; } }else{ @@ -89481,7 +93703,7 @@ SQLITE_PRIVATE void sqlite3StartTable( if( !noErr ){ sqlite3ErrorMsg(pParse, "table %T already exists", pName); }else{ - assert( !db->init.busy ); + assert( !db->init.busy || CORRUPT_DB ); sqlite3CodeVerifySchema(pParse, iDb); } goto begin_table_error; @@ -89527,10 +93749,12 @@ SQLITE_PRIVATE void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; + int addr1; int fileFormat; int reg1, reg2, reg3; - sqlite3BeginWriteOperation(pParse, 0, iDb); + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; + sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtual ){ @@ -89546,14 +93770,14 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -89574,7 +93798,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -89646,10 +93870,10 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ pCol->zName = z; /* If there is no type specified, columns have the default affinity - ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will + ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will ** be called next to set pCol->affinity correctly. */ - pCol->affinity = SQLITE_AFF_NONE; + pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; p->nCol++; } @@ -89684,7 +93908,7 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** 'CHAR' | SQLITE_AFF_TEXT ** 'CLOB' | SQLITE_AFF_TEXT ** 'TEXT' | SQLITE_AFF_TEXT -** 'BLOB' | SQLITE_AFF_NONE +** 'BLOB' | SQLITE_AFF_BLOB ** 'REAL' | SQLITE_AFF_REAL ** 'FLOA' | SQLITE_AFF_REAL ** 'DOUB' | SQLITE_AFF_REAL @@ -89710,7 +93934,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ aff = SQLITE_AFF_TEXT; }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ - aff = SQLITE_AFF_NONE; + aff = SQLITE_AFF_BLOB; if( zIn[0]=='(' ) zChar = zIn; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ @@ -89770,7 +93994,8 @@ SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; pCol = &p->aCol[p->nCol-1]; - assert( pCol->zType==0 ); + assert( pCol->zType==0 || CORRUPT_DB ); + sqlite3DbFree(pParse->db, pCol->zType); pCol->zType = sqlite3NameFromToken(pParse->db, pType); pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); } @@ -89810,6 +94035,30 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ sqlite3ExprDelete(db, pSpan->pExpr); } +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the @@ -89854,18 +94103,24 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( }else{ nTerm = pList->nExpr; for(i=0; inCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; - break; + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + break; + } } } } } if( nTerm==1 && zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC + && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -89878,14 +94133,11 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - Vdbe *v = pParse->pVdbe; Index *p; - if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); if( p ){ p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); } pList = 0; } @@ -90104,7 +94356,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_BLOB */ "", /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", @@ -90117,17 +94369,17 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); - assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_NONE ); + assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); + assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_BLOB ); testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_NONE]; + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_NONE + assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -90235,15 +94487,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; - } - - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. - */ - if( pParse->addrSkipPK ){ - assert( v ); - sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; + sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -90251,10 +94495,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token ipkToken; + ipkToken.z = pTab->aCol[pTab->iPKey].zName; + ipkToken.n = sqlite3Strlen30(ipkToken.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); @@ -90263,16 +94509,42 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); + } + + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + pPk->nColumn--; + }else{ + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; } pPk->isCovering = 1; assert( pPk!=0 ); nPk = pPk->nKeyCol; - /* Make sure every column of the PRIMARY KEY is NOT NULL */ - for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + /* Make sure every column of the PRIMARY KEY is NOT NULL. (Except, + ** do not enforce this for imposter tables.) */ + if( !db->init.imposterTable ){ + for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + } + pPk->uniqNotNull = 1; } - pPk->uniqNotNull = 1; /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; @@ -90354,9 +94626,10 @@ SQLITE_PRIVATE void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + if( pEnd==0 && pSelect==0 ){ return; } + assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -90382,7 +94655,7 @@ SQLITE_PRIVATE void sqlite3EndTable( if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); }else{ - p->tabFlags |= TF_WithoutRowid; + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; convertToWithoutRowidTable(pParse, p); } } @@ -90450,26 +94723,46 @@ SQLITE_PRIVATE void sqlite3EndTable( ** be redundant. */ if( pSelect ){ - SelectDest dest; - Table *pSelTab; - + SelectDest dest; /* Where the SELECT should store results */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int regRec; /* A record to be insert into the new table */ + int regRowid; /* Rowid of the next row to insert */ + int addrInsLoop; /* Top of the loop for inserting rows */ + Table *pSelTab; /* A table that describes the SELECT results */ + + regYield = ++pParse->nMem; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; assert(pParse->nTab==1); + sqlite3MayAbort(pParse); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; - sqlite3SelectDestInit(&dest, SRT_Table, 1); + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); + if( pParse->nErr ) return; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); + sqlite3TableAffinity(v, p, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeGoto(v, addrInsLoop); + sqlite3VdbeJumpHere(v, addrInsLoop); sqlite3VdbeAddOp1(v, OP_Close, 1); - if( pParse->nErr==0 ){ - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); - if( pSelTab==0 ) return; - assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; - p->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - } } /* Compute the complete text of the CREATE statement */ @@ -90564,6 +94857,7 @@ SQLITE_PRIVATE void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -90579,22 +94873,15 @@ SQLITE_PRIVATE void sqlite3CreateView( if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); - return; + goto create_view_fail; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -90602,30 +94889,31 @@ SQLITE_PRIVATE void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ - return; - } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + assert( sEnd.z[0]!=0 ); + if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + while( sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -90643,6 +94931,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ sqlite3_xauth xAuth; /* Saved xAuth pointer */ + u8 bEnabledLA; /* Saved db->lookaside.bEnabled state */ assert( pTable ); @@ -90688,40 +94977,46 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; + bEnabledLA = db->lookaside.bEnabled; + if( pTable->pCheck ){ db->lookaside.bEnabled = 0; + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + }else{ + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + db->lookaside.bEnabled = 0; #ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bEnabled = enableLookaside; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->schemaFlags |= DB_UnresetViews; - }else{ - pTable->nCol = 0; + pParse->nTab = n; + if( pSelTab ){ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + }else{ + pTable->nCol = 0; + nErr++; + } + sqlite3SelectDelete(db, pSel); + } else { nErr++; } - sqlite3SelectDelete(db, pSel); - } else { - nErr++; } + db->lookaside.bEnabled = bEnabledLA; + pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -90738,7 +95033,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); + sqlite3DeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -90988,6 +95283,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; @@ -91292,7 +95588,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ assert( pKey!=0 || db->mallocFailed || pParse->nErr ); if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); @@ -91301,7 +95597,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); @@ -91388,14 +95685,12 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - assert( pParse->nErr==0 ); /* Never called with prior errors */ - if( db->mallocFailed || IN_DECLARE_VTAB ){ + if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ @@ -91544,11 +95839,16 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token prevCol; + prevCol.z = pTab->aCol[pTab->nCol-1].zName; + prevCol.n = sqlite3Strlen30(prevCol.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->nCol-1].zName); - pList->a[0].sortOrder = (u8)sortOrder; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); } /* Figure out how many bytes of space are required to store explicitly @@ -91556,8 +95856,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -91598,35 +95898,54 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; - int requestedSortOrder; + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ char *zColl; /* Collation sequence name */ - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - assert( j<=0x7fff ); - pIndex->aiColumn[i] = (i16)j; - if( pListItem->pExpr ){ + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; - assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -91634,21 +95953,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = "BINARY"; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -91660,7 +95984,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = -1; + pIndex->aiColumn[i] = XN_ROWID; pIndex->azColl[i] = "BINARY"; } sqlite3DefaultRowEst(pIndex); @@ -91699,6 +96023,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; @@ -91721,6 +96046,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIdx->onError = pIndex->onError; } } + pRet = pIdx; goto exit_create_index; } } @@ -91729,6 +96055,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); @@ -91758,7 +96085,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -91766,10 +96093,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( v = sqlite3GetVdbe(pParse); if( v==0 ) goto exit_create_index; - - /* Create the rootpage for the index - */ sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); /* Gather the complete text of the CREATE INDEX statement into @@ -91809,6 +96141,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } + + sqlite3VdbeJumpHere(v, pIndex->tnum); } /* When adding an index to the list of indices for a table, make @@ -92211,7 +96545,8 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndex); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -92284,17 +96619,37 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; + pItem->fg.notIndexed = 1; }else{ - pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); } } } +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p && pList ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -92313,11 +96668,10 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; - assert( p->a || p->nSrc==0 ); for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; + p->a[i].fg.jointype = p->a[i-1].fg.jointype; } - p->a[0].jointype = 0; + p->a[0].fg.jointype = 0; } } @@ -92560,14 +96914,17 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; - for(j=0; jnKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, 0, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, 0, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, @@ -92739,40 +97096,31 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ ** when it has finished using it. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; if( pParse->nErr ) return 0; -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); } -#endif - if( pIdx->pKeyInfo==0 ){ - int i; - int nCol = pIdx->nColumn; - int nKey = pIdx->nKeyCol; - KeyInfo *pKey; - if( pIdx->uniqNotNull ){ - pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); - }else{ - pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - if( pKey ){ - assert( sqlite3KeyInfoIsWriteable(pKey) ); - for(i=0; iazColl[i]; - assert( zColl!=0 ); - pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : - sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; - } - if( pParse->nErr ){ - sqlite3KeyInfoUnref(pKey); - }else{ - pIdx->pKeyInfo = pKey; - } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + pKey = 0; } } - return sqlite3KeyInfoRef(pIdx->pKeyInfo); + return pKey; } #ifndef SQLITE_OMIT_CTE @@ -92821,7 +97169,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zErr = 0; + pNew->a[pNew->nCte].zCteErr = 0; pNew->nCte++; } @@ -92863,6 +97211,7 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ ** of user defined functions and collation sequences. */ +/* #include "sqliteInt.h" */ /* ** Invoke the 'collation needed' callback to request a collation sequence @@ -93340,6 +97689,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** This file contains C code routines that are called by the parser ** in order to generate code for DELETE FROM statements. */ +/* #include "sqliteInt.h" */ /* ** While a SrcList can in general represent multiple tables and subqueries @@ -93516,7 +97866,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( pInClause->x.pSelect = pSelect; pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeight(pParse, pInClause); + sqlite3ExprSetHeightAndFlags(pParse, pInClause); return pInClause; /* something went wrong. clean up anything allocated. */ @@ -93553,8 +97903,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ int iTabCur; /* Cursor number for the table */ - int iDataCur; /* VDBE cursor for the canonical data source */ - int iIdxCur; /* Cursor number of the first index */ + int iDataCur = 0; /* VDBE cursor for the canonical data source */ + int iIdxCur = 0; /* Cursor number of the first index */ int nIdx; /* Number of indices */ sqlite3 *db; /* Main database structure */ AuthContext sContext; /* Authorization context */ @@ -93562,7 +97912,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -93574,12 +97924,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ - int addrDelete = 0; /* Jump directly to the delete logic */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ + int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -93603,9 +97953,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 +# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -93686,8 +98038,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -93702,6 +98056,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -93722,13 +98078,18 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -93738,6 +98099,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -93748,11 +98110,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ aToOpen = sqlite3DbMallocRaw(db, nIdx+2); if( aToOpen==0 ){ @@ -93764,27 +98125,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass!=ONEPASS_OFF ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -93793,21 +98154,24 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ + if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - assert( !IsVirtual(pTab) ); - if( aToOpen[iDataCur-iTabCur] ){ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); @@ -93829,23 +98193,32 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } }else #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } @@ -93912,6 +98285,25 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -93923,7 +98315,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -93940,7 +98333,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -94000,11 +98393,15 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** a view (in which case the only effect of the DELETE statement is to ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + sqlite3VdbeChangeP5(v, eMode==ONEPASS_MULTI); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -94047,7 +98444,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -94063,11 +98461,12 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -94116,17 +98515,16 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); }else{ *piPartIdxLabel = 0; } @@ -94135,9 +98533,14 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -94182,14 +98585,18 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ ** functions of SQLite. (Some function, and in particular the date and ** time functions, are implemented separately.) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ +/* #include "vdbeInt.h" */ /* ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1]; + VdbeOp *pOp; + assert( context->pVdbe!=0 ); + pOp = &context->pVdbe->aOp[context->iOp-1]; assert( pOp->opcode==OP_CollSeq ); assert( pOp->p4type==P4_COLLSEQ ); return pOp->p4.pColl; @@ -94324,8 +98731,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ default: { /* Because sqlite3_value_double() returns 0.0 if the argument is not ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal; @@ -94397,13 +98804,13 @@ static void printfFunc( StrAccum str; const char *zFormat; int n; + sqlite3 *db = sqlite3_context_db_handle(context); if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ x.nArg = argc-1; x.nUsed = 0; x.apArg = argv+1; - sqlite3StrAccumInit(&str, 0, 0, SQLITE_MAX_LENGTH); - str.db = sqlite3_context_db_handle(context); + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, @@ -94458,6 +98865,14 @@ static void substrFunc( } } } +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + if( p1==0 ) p1 = 1; /* */ +#endif if( argc==3 ){ p2 = sqlite3_value_int(argv[2]); if( p2<0 ){ @@ -94545,7 +98960,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ #endif /* -** Allocate nByte bytes of space using sqlite3_malloc(). If the +** Allocate nByte bytes of space using sqlite3Malloc(). If the ** allocation fails, call sqlite3_result_error_nomem() to notify ** the database handle that malloc() has failed and return NULL. ** If nByte is larger than the maximum string or blob length, then @@ -94732,17 +99147,15 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, all characters are -** able to participate in upper-case-to-lower-case mappings in EBCDIC -** whereas only characters less than 0x80 do in ASCII. +** character is exactly one byte in size. Also, provde the Utf8Read() +** macro for fast reading of the next character in the common case where +** the next character is ASCII. */ #if defined(SQLITE_EBCDIC) # define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] -# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] +# define Utf8Read(A) (*(A++)) #else -# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } -# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] +# define Utf8Read(A) (A[0]<0x80?*(A++):sqlite3Utf8Read(&A)) #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -94784,7 +99197,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** Ec Where E is the "esc" character and c is any other ** character, including '%', '_', and esc, match exactly c. ** -** The comments through this routine usually assume glob matching. +** The comments within this routine usually assume glob matching. ** ** This routine is usually quick, but can be N**2 in the worst case. */ @@ -94808,13 +99221,12 @@ static int patternCompare( */ matchOther = esc ? esc : pInfo->matchSet; - while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + while( (c = Utf8Read(zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there ** are also "?" characters, skip those as well, but consume a ** single character of the input string for each "?" skipped */ - while( (c=sqlite3Utf8Read(&zPattern)) == matchAll - || c == matchOne ){ + while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } @@ -94859,7 +99271,7 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } }else{ - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + while( (c2 = Utf8Read(zString))!=0 ){ if( c2!=c ) continue; if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } @@ -94905,7 +99317,7 @@ static int patternCompare( continue; } } - c2 = sqlite3Utf8Read(&zString); + c2 = Utf8Read(zString); if( c==c2 ) continue; if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ continue; @@ -94919,7 +99331,7 @@ static int patternCompare( /* ** The sqlite3_strglob() interface. */ -SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ +SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlobPattern, const char *zString){ return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; } @@ -95214,7 +99626,7 @@ static void charFunc( ){ unsigned char *z, *zOut; int i; - zOut = z = sqlite3_malloc( argc*4+1 ); + zOut = z = sqlite3_malloc64( argc*4+1 ); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -95282,16 +99694,14 @@ static void zeroblobFunc( sqlite3_value **argv ){ i64 n; - sqlite3 *db = sqlite3_context_db_handle(context); + int rc; assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - }else{ - sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + if( n<0 ) n = 0; + rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ + if( rc ){ + sqlite3_result_error_code(context, rc); } } @@ -95362,7 +99772,7 @@ static void replaceFunc( return; } zOld = zOut; - zOut = sqlite3_realloc(zOut, (int)nOut); + zOut = sqlite3_realloc64(zOut, (int)nOut); if( zOut==0 ){ sqlite3_result_error_nomem(context); sqlite3_free(zOld); @@ -95724,8 +100134,7 @@ static void groupConcatStep( if( pAccum ){ sqlite3 *db = sqlite3_context_db_handle(context); - int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 2; + int firstTerm = pAccum->mxAlloc==0; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ @@ -95809,6 +100218,11 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) ** then set aWc[0] through aWc[2] to the wildcard characters and ** return TRUE. If the function is not a LIKE-style function then ** return FALSE. +** +** *pIsNocase is set to true if uppercase and lowercase are equivalent for +** the function (default for LIKE). If the function makes the distinction +** between uppercase and lowercase (as does GLOB) then *pIsNocase is set to +** false. */ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ FuncDef *pDef; @@ -95895,15 +100309,15 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #if SQLITE_USER_AUTHENTICATION FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), #endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), @@ -95915,8 +100329,8 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), @@ -95968,6 +100382,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** This file contains code used by the compiler to add foreign key ** support to compiled SQL statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_FOREIGN_KEY #ifndef SQLITE_OMIT_TRIGGER @@ -96208,6 +100623,8 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ @@ -96330,7 +100747,7 @@ static void fkLookupParent( sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); sqlite3ReleaseTempReg(pParse, regTemp); @@ -96360,6 +100777,7 @@ static void fkLookupParent( for(i=0; iaiColumn[i]+1+regData; + assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ @@ -96368,11 +100786,11 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); @@ -96393,7 +100811,7 @@ static void fkLookupParent( OE_Abort, 0, P4_STATIC, P5_ConstraintFK); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); } @@ -96465,6 +100883,10 @@ static Expr *exprTableColumn( ** code for an SQL UPDATE operation, this function may be called twice - ** once to "delete" the old row and once to "insert" the new row. ** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** ** The code generated by this function scans through the rows in the child ** table that correspond to the parent table row being deleted or inserted. ** For each child row found, one of the following actions is taken: @@ -96564,6 +100986,7 @@ static void fkScanChildren( assert( pIdx!=0 ); for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); @@ -96581,13 +101004,9 @@ static void fkScanChildren( sqlite3ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE - ** clause. If the constraint is not deferred, throw an exception for - ** each row found. Otherwise, for deferred constraints, increment the - ** deferred constraint counter by nIncr for each row selected. */ + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; - } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); @@ -96766,6 +101185,24 @@ static int fkParentIsModified( return 0; } +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + /* ** This function is called when inserting, deleting or updating a row of ** table pTab to generate VDBE code to perform foreign key constraint @@ -96818,7 +101255,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( int *aiCol; int iCol; int i; - int isIgnore = 0; + int bIgnore = 0; if( aChange && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 @@ -96869,6 +101306,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any @@ -96877,7 +101315,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( int rcauth; char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); - isIgnore = (rcauth==SQLITE_IGNORE); + bIgnore = (rcauth==SQLITE_IGNORE); } #endif } @@ -96892,12 +101330,18 @@ SQLITE_PRIVATE void sqlite3FkCheck( /* A row is being removed from the child table. Search for the parent. ** If the parent does not exist, removing the child row resolves an ** outstanding foreign key constraint violation. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); } - if( regNew!=0 ){ + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As all child key + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); } sqlite3DbFree(db, aiFree); @@ -96918,8 +101362,8 @@ SQLITE_PRIVATE void sqlite3FkCheck( && !pParse->pToplevel && !pParse->isMultiWrite ){ assert( regOld==0 && regNew!=0 ); - /* Inserting a single row into a parent table cannot cause an immediate - ** foreign key violation. So do nothing in this case. */ + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ continue; } @@ -96943,13 +101387,28 @@ SQLITE_PRIVATE void sqlite3FkCheck( fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } if( regOld!=0 ){ - /* If there is a RESTRICT action configured for the current operation - ** on the parent table of this FK, then throw an exception - ** immediately if the FK constraint is violated, even if this is a - ** deferred trigger. That's what RESTRICT means. To defer checking - ** the constraint, the FK should specify NO ACTION (represented - ** using OE_None). NO ACTION is the default. */ + int eAction = pFKey->aAction[aChange!=0]; fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } } pItem->zName = 0; sqlite3SrcListDelete(db, pSrc); @@ -96979,7 +101438,10 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } } } @@ -97101,7 +101563,9 @@ static Trigger *fkActionTrigger( iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); - tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; + assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); + tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; tToCol.n = sqlite3Strlen30(tToCol.z); @@ -97113,10 +101577,10 @@ static Trigger *fkActionTrigger( ** parent table are used for the comparison. */ pEq = sqlite3PExpr(pParse, TK_EQ, sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) , 0), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tFromCol) + sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) , 0); pWhere = sqlite3ExprAnd(db, pWhere, pEq); @@ -97128,12 +101592,12 @@ static Trigger *fkActionTrigger( if( pChanges ){ pEq = sqlite3PExpr(pParse, TK_IS, sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), 0), sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), 0), 0); pWhen = sqlite3ExprAnd(db, pWhen, pEq); @@ -97143,8 +101607,8 @@ static Trigger *fkActionTrigger( Expr *pNew; if( action==OE_Cascade ){ pNew = sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) , 0); }else if( action==OE_SetDflt ){ Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; @@ -97191,13 +101655,12 @@ static Trigger *fkActionTrigger( pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->target.z */ + nFrom + 1 /* Space for pStep->zTarget */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->target.z = (char *)&pStep[1]; - pStep->target.n = nFrom; - memcpy((char *)pStep->target.z, zFrom, nFrom); + pStep->zTarget = (char *)&pStep[1]; + memcpy((char *)pStep->zTarget, zFrom, nFrom); pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); @@ -97333,6 +101796,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Generate code that will @@ -97362,7 +101826,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); - assert( pPk->tnum=pTab->tnum ); + assert( pPk->tnum==pTab->tnum ); sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pPk); VdbeComment((v, "%s", pTab->zName)); @@ -97376,7 +101840,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'A' NONE +** 'A' BLOB ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -97389,7 +101853,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -97401,7 +101865,6 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ db->mallocFailed = 1; @@ -97409,7 +101872,18 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + char aff; + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; + } } pIdx->zColAff[n] = 0; } @@ -97419,9 +101893,9 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ /* ** Compute the affinity string for table pTab, if it has not already been -** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and ** if iReg>0 then code an OP_Affinity opcode that will set the affinities ** for register iReg and following. Or if affinities exists and iReg==0, ** then just set the P4 operand of the previous opcode (which should be @@ -97431,7 +101905,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'A' NONE +** 'A' BLOB ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -97453,7 +101927,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } do{ zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); + }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } i = sqlite3Strlen30(zColAff); @@ -97570,7 +102044,7 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + assert( sqlite3IsToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ @@ -97580,14 +102054,14 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeGoto(v, addr+9); sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); sqlite3VdbeAddOp0(v, OP_Close); @@ -97623,16 +102097,16 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ Db *pDb = &db->aDb[p->iDb]; - int j1; + int addr1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -97662,20 +102136,23 @@ static int xferOptimization( /* ** This routine is called to handle SQL of the following forms: ** -** insert into TABLE (IDLIST) values(EXPRLIST) +** insert into TABLE (IDLIST) values(EXPRLIST),(EXPRLIST),... ** insert into TABLE (IDLIST) select +** insert into TABLE (IDLIST) default values ** ** The IDLIST following the table name is always optional. If omitted, -** then a list of all columns for the table is substituted. The IDLIST -** appears in the pColumn parameter. pColumn is NULL if IDLIST is omitted. +** then a list of all (non-hidden) columns for the table is substituted. +** The IDLIST appears in the pColumn parameter. pColumn is NULL if IDLIST +** is omitted. ** -** The pList parameter holds EXPRLIST in the first form of the INSERT -** statement above, and pSelect is NULL. For the second form, pList is -** NULL and pSelect is a pointer to the select statement used to generate -** data for the insert. +** For the pSelect parameter holds the values to be inserted for the +** first two forms shown above. A VALUES clause is really just short-hand +** for a SELECT statement that omits the FROM clause and everything else +** that follows. If the pSelect parameter is NULL, that means that the +** DEFAULT VALUES form of the INSERT statement is intended. ** ** The code generated follows one of four templates. For a simple -** insert with data coming from a VALUES clause, the code executes +** insert with data coming from a single-row VALUES clause, the code executes ** once straight down through. Pseudo-code follows (we call this ** the "1st template"): ** @@ -97782,7 +102259,7 @@ SQLITE_PRIVATE void sqlite3Insert( u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ - u8 bIdListInOrder = 1; /* True if IDLIST is in table order */ + u8 bIdListInOrder; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ @@ -97807,8 +102284,8 @@ SQLITE_PRIVATE void sqlite3Insert( } /* If the Select object is really just a simple VALUES() list with a - ** single row values (the common case) then keep that one row of values - ** and go ahead and discard the Select object + ** single row (the common case) then keep that one row of values + ** and discard the other (unused) parts of the pSelect object */ if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ pList = pSelect->pEList; @@ -97916,6 +102393,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** is appears in the original table. (The index of the INTEGER ** PRIMARY KEY in the original table is pTab->iPKey.) */ + bIdListInOrder = (pTab->tabFlags & TF_OOOHidden)==0; if( pColumn ){ for(i=0; inId; i++){ pColumn->a[i].idx = -1; @@ -97951,7 +102429,8 @@ SQLITE_PRIVATE void sqlite3Insert( ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate a co-routine to run the SELECT */ + /* Data is coming from a SELECT or from a multi-row VALUES clause. + ** Generate a co-routine to run the SELECT. */ int regYield; /* Register holding co-routine entry-point */ int addrTop; /* Top of the co-routine */ int rc; /* Result code */ @@ -97964,8 +102443,7 @@ SQLITE_PRIVATE void sqlite3Insert( dest.nSdst = pTab->nCol; rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; - assert( pParse->nErr==0 || rc ); - if( rc || db->mallocFailed ) goto insert_cleanup; + if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); @@ -98007,25 +102485,27 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeGoto(v, addrL); sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); } }else{ - /* This is the case if the data for the INSERT is coming from a VALUES - ** clause + /* This is the case if the data for the INSERT is coming from a + ** single-row VALUES clause */ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; ia[i].pExpr) ){ + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ goto insert_cleanup; } + }else{ + nColumn = 0; } } @@ -98118,7 +102598,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int j1; + int addr1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); @@ -98126,9 +102606,9 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } @@ -98202,14 +102682,14 @@ SQLITE_PRIVATE void sqlite3Insert( ** to generate a unique primary key value. */ if( !appendFlag ){ - int j1; + int addr1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } @@ -98306,7 +102786,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeGoto(v, addrCont); sqlite3VdbeJumpHere(v, addrInsTop); } @@ -98463,7 +102943,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Address of jump instruction */ + int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -98534,9 +103014,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); break; } } @@ -98553,7 +103034,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int allOk = sqlite3VdbeMakeLabel(v); sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; @@ -98651,17 +103132,20 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + regNewData, 1, 0, OE_Replace, + ONEPASS_SINGLE, -1); + }else{ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } } seenReplace = 1; break; } case OE_Ignore: { /*assert( seenReplace==0 );*/ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } } @@ -98697,8 +103181,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); pParse->ckBase = regNewData+1; - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); pParse->ckBase = 0; } @@ -98709,15 +103193,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==XN_EXPR ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==XN_ROWID || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -98767,6 +103258,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, @@ -98788,6 +103280,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; + assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; @@ -98814,7 +103307,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } default: { @@ -98825,7 +103318,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); seenReplace = 1; break; } @@ -98835,7 +103329,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeGoto(v, ipkTop+1); sqlite3VdbeJumpHere(v, ipkBottom); } @@ -99038,6 +103532,13 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } @@ -99085,6 +103586,7 @@ static int xferOptimization( int onError, /* How to handle constraint errors */ int iDbDest /* The database of pDest */ ){ + sqlite3 *db = pParse->db; ExprList *pEList; /* The result set of the SELECT */ Table *pSrc; /* The table in the FROM clause of SELECT */ Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ @@ -99232,11 +103734,11 @@ static int xferOptimization( ** the extra complication to make this rule less restrictive is probably ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] */ - if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ return 0; } #endif - if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ + if( (db->flags & SQLITE_CountRows)!=0 ){ return 0; /* xfer opt does not play well with PRAGMA count_changes */ } @@ -99247,7 +103749,7 @@ static int xferOptimization( #ifdef SQLITE_TEST sqlite3_xferopt_count++; #endif - iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); + iDbSrc = sqlite3SchemaToIndex(db, pSrc->pSchema); v = sqlite3GetVdbe(pParse); sqlite3CodeVerifySchema(pParse, iDbSrc); iSrc = pParse->nTab++; @@ -99257,14 +103759,18 @@ static int xferOptimization( regRowid = sqlite3GetTempReg(pParse); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); assert( HasRowid(pDest) || destHasUniqueIdx ); - if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + if( (db->flags & SQLITE_Vacuum)==0 && ( + (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ || destHasUniqueIdx /* (2) */ || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ - ){ + )){ /* In some circumstances, we are able to run the xfer optimization - ** only if the destination table is initially empty. This code makes - ** that determination. Conditions under which the destination must - ** be empty: + ** only if the destination table is initially empty. Unless the + ** SQLITE_Vacuum flag is set, this block generates code to make + ** that determination. If SQLITE_Vacuum is set, then the destination + ** table is always empty. + ** + ** Conditions under which the destination must be empty: ** ** (1) There is no INTEGER PRIMARY KEY but there are indices. ** (If the destination is not initially empty, the rowid fields @@ -99276,7 +103782,7 @@ static int xferOptimization( ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ @@ -99307,6 +103813,7 @@ static int xferOptimization( sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + u8 idxInsFlags = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } @@ -99320,7 +103827,36 @@ static int xferOptimization( VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + if( db->flags & SQLITE_Vacuum ){ + /* This INSERT command is part of a VACUUM operation, which guarantees + ** that the destination table is empty. If all indexed columns use + ** collation sequence BINARY, then it can also be assumed that the + ** index will be populated by inserting keys in strictly sorted + ** order. In this case, instead of seeking within the b-tree as part + ** of every OP_IdxInsert opcode, an OP_Last is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key + ** should be inserted. This is faster. + ** + ** If any of the indexed columns use a collation sequence other than + ** BINARY, this optimization is disabled. This is because the user + ** might change the definition of a collation sequence and then run + ** a VACUUM command. In that case keys may not be written in strictly + ** sorted order. */ + for(i=0; inColumn; i++){ + char *zColl = pSrcIdx->azColl[i]; + assert( zColl!=0 ); + if( sqlite3_stricmp("BINARY", zColl) ) break; + } + if( i==pSrcIdx->nColumn ){ + idxInsFlags = OPFLAG_USESEEKRESULT; + sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + } + } + if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + idxInsFlags |= OPFLAG_NCHANGE; + } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeChangeP5(v, idxInsFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -99359,6 +103895,7 @@ static int xferOptimization( ** accessed by users of the library. */ +/* #include "sqliteInt.h" */ /* ** Execute SQL code. Return one of the SQLITE_ success/failure @@ -99370,7 +103907,7 @@ static int xferOptimization( ** argument to xCallback(). If xCallback=NULL then no callback ** is invoked, even for queries. */ -SQLITE_API int sqlite3_exec( +SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ sqlite3_callback xCallback, /* Invoke this callback routine */ @@ -99527,6 +104064,7 @@ SQLITE_API int sqlite3_exec( */ #ifndef _SQLITE3EXT_H_ #define _SQLITE3EXT_H_ +/* #include "sqlite3.h" */ typedef struct sqlite3_api_routines sqlite3_api_routines; @@ -99776,6 +104314,14 @@ struct sqlite3_api_routines { void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, void(*)(void*), unsigned char); int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); }; /* @@ -99789,7 +104335,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -99916,6 +104462,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -100006,9 +104553,17 @@ struct sqlite3_api_routines { #define sqlite3_result_blob64 sqlite3_api->result_blob64 #define sqlite3_result_text64 sqlite3_api->result_text64 #define sqlite3_strglob sqlite3_api->strglob -#endif /* SQLITE_CORE */ - -#ifndef SQLITE_CORE +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; @@ -100027,6 +104582,7 @@ struct sqlite3_api_routines { /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ +/* #include "sqliteInt.h" */ /* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -100043,7 +104599,6 @@ struct sqlite3_api_routines { # define sqlite3_column_table_name16 0 # define sqlite3_column_origin_name 0 # define sqlite3_column_origin_name16 0 -# define sqlite3_table_column_metadata 0 #endif #ifdef SQLITE_OMIT_AUTHORIZATION @@ -100412,7 +104967,15 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_reset_auto_extension, sqlite3_result_blob64, sqlite3_result_text64, - sqlite3_strglob + sqlite3_strglob, + /* Version 3.8.11 and later */ + (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, + sqlite3_value_free, + sqlite3_result_zeroblob64, + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype }; /* @@ -100440,7 +105003,7 @@ static int sqlite3LoadExtension( const char *zEntry; char *zAltEntry = 0; void **aHandle; - int nMsg = 300 + sqlite3Strlen30(zFile); + u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; /* Shared library endings to try if zFile cannot be loaded as written */ @@ -100483,7 +105046,7 @@ static int sqlite3LoadExtension( #endif if( handle==0 ){ if( pzErrMsg ){ - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "unable to open shared library [%s]", zFile); @@ -100509,7 +105072,7 @@ static int sqlite3LoadExtension( if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); - zAltEntry = sqlite3_malloc(ncFile+30); + zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM; @@ -100531,7 +105094,7 @@ static int sqlite3LoadExtension( if( xInit==0 ){ if( pzErrMsg ){ nMsg += sqlite3Strlen30(zEntry); - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zEntry, zFile); @@ -100566,7 +105129,7 @@ static int sqlite3LoadExtension( db->aExtension[db->nExtension++] = handle; return SQLITE_OK; } -SQLITE_API int sqlite3_load_extension( +SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ @@ -100597,7 +105160,7 @@ SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ ** Enable or disable extension loading. Extension loading is disabled by ** default so as not to open security holes in older applications. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ +SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ db->flags |= SQLITE_LoadExtension; @@ -100617,7 +105180,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ ** dummy pointer. */ #ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; +static const sqlite3_api_routines sqlite3Apis; #endif @@ -100630,7 +105193,7 @@ static const sqlite3_api_routines sqlite3Apis = { 0 }; */ typedef struct sqlite3AutoExtList sqlite3AutoExtList; static SQLITE_WSD struct sqlite3AutoExtList { - int nExt; /* Number of entries in aExt[] */ + u32 nExt; /* Number of entries in aExt[] */ void (**aExt)(void); /* Pointers to the extension init functions */ } sqlite3Autoext = { 0, 0 }; @@ -100654,7 +105217,7 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ +SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -100663,7 +105226,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ }else #endif { - int i; + u32 i; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -100673,9 +105236,9 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ if( wsdAutoext.aExt[i]==xInit ) break; } if( i==wsdAutoext.nExt ){ - int nByte = (wsdAutoext.nExt+1)*sizeof(wsdAutoext.aExt[0]); + u64 nByte = (wsdAutoext.nExt+1)*sizeof(wsdAutoext.aExt[0]); void (**aNew)(void); - aNew = sqlite3_realloc(wsdAutoext.aExt, nByte); + aNew = sqlite3_realloc64(wsdAutoext.aExt, nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -100699,7 +105262,7 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ +SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xInit)(void)){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -100707,7 +105270,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ int n = 0; wsdAutoextInit; sqlite3_mutex_enter(mutex); - for(i=wsdAutoext.nExt-1; i>=0; i--){ + for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ wsdAutoext.nExt--; wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; @@ -100722,7 +105285,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ /* ** Reset the automatic extension loading mechanism. */ -SQLITE_API void sqlite3_reset_auto_extension(void){ +SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize()==SQLITE_OK ) #endif @@ -100745,7 +105308,7 @@ SQLITE_API void sqlite3_reset_auto_extension(void){ ** If anything goes wrong, set an error in the database connection. */ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ - int i; + u32 i; int go = 1; int rc; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); @@ -100794,6 +105357,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ************************************************************************* ** This file contains code used to implement the PRAGMA command. */ +/* #include "sqliteInt.h" */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -100804,11 +105368,18 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #endif /*************************************************************************** -** The next block of code, including the PragTyp_XXXX macro definitions and -** the aPragmaName[] object is composed of generated code. DO NOT EDIT. -** -** To add new pragmas, edit the code in ../tool/mkpragmatab.tcl and rerun -** that script. Then copy/paste the output in place of the following: +** The "pragma.h" include file is an automatically generated file that +** that includes the PragType_XXXX macro definitions and the aPragmaName[] +** object. This ensures that the aPragmaName[] table is arranged in +** lexicographical order to facility a binary search of the pragma name. +** Do not edit pragma.h directly. Edit and rerun the script in at +** ../tool/mkpragmatab.tcl. */ +/************** Include pragma.h in the middle of pragma.c *******************/ +/************** Begin file pragma.h ******************************************/ +/* DO NOT EDIT! +** This file is automatically generated by the script at +** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit +** that script and rerun it. */ #define PragTyp_HEADER_VALUE 0 #define PragTyp_AUTO_VACUUM 1 @@ -100853,6 +105424,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #define PragTyp_LOCK_STATUS 40 #define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 +#define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ u8 ePragTyp; /* PragTyp_XXX value */ @@ -100869,7 +105441,7 @@ static const struct sPragmaNames { { /* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) { /* zName: */ "auto_vacuum", @@ -100892,7 +105464,7 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, + /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -100905,6 +105477,10 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlag: */ 0, /* iArg: */ 0 }, + { /* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "checkpoint_fullfsync", /* ePragTyp: */ PragTyp_FLAG, @@ -100935,6 +105511,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_DATA_VERSION }, +#endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) { /* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, @@ -100990,8 +105572,8 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "freelist_count", /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "full_column_names", @@ -101036,6 +105618,10 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlag: */ PragFlag_NeedSchema, /* iArg: */ 0 }, + { /* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) { /* zName: */ "integrity_check", @@ -101143,7 +105729,7 @@ static const struct sPragmaNames { { /* zName: */ "schema_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "secure_delete", @@ -101209,7 +105795,7 @@ static const struct sPragmaNames { { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) @@ -101252,9 +105838,10 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 57 on by default, 70 total. */ -/* End of the automatically generated pragma table. -***************************************************************************/ +/* Number of pragmas: 60 on by default, 73 total. */ + +/************** End of pragma.h **********************************************/ +/************** Continuing where we left off in pragma.c *********************/ /* ** Interpret the given string as a safety level. Return 0 for OFF, @@ -101385,20 +105972,46 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ } #endif /* SQLITE_PAGER_PRAGMAS */ +/* +** Set the names of the first N columns to the values in azCol[] +*/ +static void setAllColumnNames( + Vdbe *v, /* The query under construction */ + int N, /* Number of columns */ + const char **azCol /* Names of columns */ +){ + int i; + sqlite3VdbeSetNumCols(v, N); + for(i=0; inMem; - i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); - if( pI64 ){ - memcpy(pI64, &value, sizeof(value)); +static void returnSingleInt(Vdbe *v, const char *zLabel, i64 value){ + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, 1, 0, (const u8*)&value, P4_INT64); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + +/* +** Generate code to return a single text value. +*/ +static void returnSingleText( + Vdbe *v, /* Prepared statement under construction */ + const char *zLabel, /* Name of the result column */ + const char *zValue /* Value to be returned */ +){ + if( zValue ){ + sqlite3VdbeLoadString(v, 1, (const char*)zValue); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - sqlite3VdbeAddOp4(v, OP_Int64, 0, mem, 0, (char*)pI64, P4_INT64); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); } @@ -101502,11 +106115,12 @@ SQLITE_PRIVATE void sqlite3Pragma( Token *pId; /* Pointer to token */ char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int iDb; /* Database index for */ - int lwr, upr, mid; /* Binary search bounds */ + int lwr, upr, mid = 0; /* Binary search bounds */ int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + const struct sPragmaNames *pPragma; if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -101542,6 +106156,17 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS ** connection. If it returns SQLITE_OK, then assume that the VFS ** handled the pragma and generate a no-op prepared statement. + ** + ** IMPLEMENTATION-OF: R-12238-55120 Whenever a PRAGMA statement is parsed, + ** an SQLITE_FCNTL_PRAGMA file control is sent to the open sqlite3_file + ** object corresponding to the database file to which the pragma + ** statement refers. + ** + ** IMPLEMENTATION-OF: R-29875-31678 The argument to the SQLITE_FCNTL_PRAGMA + ** file control is an array of pointers to strings (char**) in which the + ** second element of the array is the name of the pragma and the third + ** element is the argument to the pragma or NULL if the pragma has no + ** argument. */ aFcntl[0] = 0; aFcntl[1] = zLeft; @@ -101550,14 +106175,8 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - if( aFcntl[0] ){ - int mem = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); - sqlite3_free(aFcntl[0]); - } + returnSingleText(v, "result", aFcntl[0]); + sqlite3_free(aFcntl[0]); goto pragma_out; } if( rc!=SQLITE_NOTFOUND ){ @@ -101584,14 +106203,15 @@ SQLITE_PRIVATE void sqlite3Pragma( } } if( lwr>upr ) goto pragma_out; + pPragma = &aPragmaNames[mid]; /* Make sure the database schema is loaded if the pragma requires that */ - if( (aPragmaNames[mid].mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( (pPragma->mPragFlag & PragFlag_NeedSchema)!=0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; } /* Jump to the appropriate pragma handler */ - switch( aPragmaNames[mid].ePragTyp ){ + switch( pPragma->ePragTyp ){ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* @@ -101626,8 +106246,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int addr; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + setOneColumnName(v, "cache_size"); pParse->nMem += 2; addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); sqlite3VdbeChangeP1(v, addr, iDb); @@ -101661,7 +106280,7 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(pParse, "page_size", size); + returnSingleInt(v, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). @@ -101696,7 +106315,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); + returnSingleInt(v, "secure_delete", b); break; } @@ -101775,10 +106394,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "locking_mode", zRet); break; } @@ -101791,9 +106407,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); - + setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -101839,7 +106453,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(pParse, "journal_size_limit", iLimit); + returnSingleInt(v, "journal_size_limit", iLimit); break; } @@ -101857,7 +106471,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -101934,11 +106548,13 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + if( sqlite3ReadSchema(pParse) ) goto pragma_out; } break; } @@ -101979,7 +106595,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(pParse, "mmap_size", sz); + returnSingleInt(v, "mmap_size", sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -102000,7 +106616,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(pParse, "temp_store", db->temp_store); + returnSingleInt(v, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -102019,13 +106635,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_temp_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -102069,13 +106679,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_data_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "data_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "data_store_directory", sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -102114,14 +106718,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - - if( proxy_file_path ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "lock_proxy_file", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "lock_proxy_file", proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -102153,13 +106750,15 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + returnSingleInt(v, "synchronous", pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); }else{ - pDb->safety_level = getSafetyLevel(zRight,0,1)+1; + int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; + if( iLevel==0 ) iLevel = 1; + pDb->safety_level = iLevel; setAllPagerFlags(db); } } @@ -102170,10 +106769,9 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(pParse, aPragmaNames[mid].zName, - (db->flags & aPragmaNames[mid].iArg)!=0 ); + returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); }else{ - int mask = aPragmaNames[mid].iArg; /* Mask of bits to set or clear. */ + int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ /* Foreign key support may not be enabled or disabled while not ** in auto-commit mode. */ @@ -102221,43 +106819,36 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "cid", "name", "type", "notnull", "dflt_value", "pk" + }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ if( IsHiddenColumn(pCol) ){ nHidden++; continue; } - sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); - if( pCol->zDflt ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, 5); - } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ k = 1; }else{ - for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} - } - sqlite3VdbeAddOp2(v, OP_Integer, k, 6); + for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeMultiLoad(v, 1, "issisi", + i-nHidden, + pCol->zName, + pCol->zType ? pCol->zType : "", + pCol->notNull ? 1 : 0, + pCol->zDflt, + k); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -102265,31 +106856,26 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_STATS: { + static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Null, 0, 2); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeMultiLoad(v, 1, "ssii", + pTab->zName, + 0, + (int)sqlite3LogEstToInt(pTab->szTabRow), + (int)sqlite3LogEstToInt(pTab->nRowLogEst)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeMultiLoad(v, 2, "sii", + pIdx->zName, + (int)sqlite3LogEstToInt(pIdx->szIdxRow), + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0])); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -102301,21 +106887,35 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ + static const char *azCol[] = { + "seqno", "cid", "name", "desc", "coll", "key" + }; int i; + int mx; + if( pPragma->iArg ){ + /* PRAGMA index_xinfo (newer version with more rows and columns) */ + mx = pIdx->nColumn; + pParse->nMem = 6; + }else{ + /* PRAGMA index_info (legacy version) */ + mx = pIdx->nKeyCol; + pParse->nMem = 3; + } pTab = pIdx->pTable; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - for(i=0; inKeyCol; i++){ + assert( pParse->nMem<=ArraySize(azCol) ); + setAllColumnNames(v, pParse->nMem, azCol); + for(i=0; iaiColumn[i]; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); - assert( pTab->nCol>cnum ); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zName); + if( pPragma->iArg ){ + sqlite3VdbeMultiLoad(v, 4, "isi", + pIdx->aSortOrder[i], + pIdx->azColl[i], + inKeyCol); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); } } } @@ -102327,53 +106927,53 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "seq", "name", "unique", "origin", "partial" + }; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + const char *azOrigin[] = { "c", "u", "pk" }; + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; case PragTyp_DATABASE_LIST: { + static const char *azCol[] = { "seq", "name", "file" }; int i; - sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { + static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; - sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } @@ -102389,33 +106989,26 @@ SQLITE_PRIVATE void sqlite3Pragma( v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ + static const char *azCol[] = { + "id", "seq", "table", "from", "to", "on_update", "on_delete", + "match" + }; int i = 0; - sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); while(pFK){ int j; for(j=0; jnCol; j++){ - char *zCol = pFK->aCol[j].zCol; - char *zOnDelete = (char *)actionName(pFK->aAction[0]); - char *zOnUpdate = (char *)actionName(pFK->aAction[1]); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, j, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; @@ -102444,17 +107037,14 @@ SQLITE_PRIVATE void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ + static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -102469,8 +107059,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, - P4_TRANSIENT); + sqlite3VdbeLoadString(v, regResult, pTab->zName); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; @@ -102515,7 +107104,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; jnCol; j++){ @@ -102525,15 +107114,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, - pFK->zTo, P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -102587,8 +107174,9 @@ SQLITE_PRIVATE void sqlite3Pragma( */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_IfNeg, 1, 0, 0}, /* 0 */ - { OP_String8, 0, 3, 0}, /* 1 */ + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, }; @@ -102609,8 +107197,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -102732,13 +107319,11 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - " missing from index ", P4_STATIC); + sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); @@ -102753,20 +107338,19 @@ SQLITE_PRIVATE void sqlite3Pragma( int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol>=0 && iColnCol ); - if( pTab->aCol[iCol].notNull ) continue; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeGoto(v, uniqOk); sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - "non-unique entry in index ", P4_STATIC); - sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); } sqlite3VdbeJumpHere(v, jmp4); @@ -102775,8 +107359,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, - "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); @@ -102786,7 +107369,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeLoadString(v, 3, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } @@ -102794,9 +107377,9 @@ SQLITE_PRIVATE void sqlite3Pragma( } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP3(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + sqlite3VdbeChangeP2(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr+1); + sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -102842,14 +107425,10 @@ SQLITE_PRIVATE void sqlite3Pragma( const struct EncName *pEnc; if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_String8, 0, 1); assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -102862,7 +107441,8 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = ENC(db) = + pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; break; } } @@ -102907,24 +107487,9 @@ SQLITE_PRIVATE void sqlite3Pragma( ** applications for any purpose. */ case PragTyp_HEADER_VALUE: { - int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + int iCookie = pPragma->iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - switch( zLeft[0] ){ - case 'a': case 'A': - iCookie = BTREE_APPLICATION_ID; - break; - case 'f': case 'F': - iCookie = BTREE_FREE_PAGE_COUNT; - break; - case 's': case 'S': - iCookie = BTREE_SCHEMA_VERSION; - break; - default: - iCookie = BTREE_USER_VERSION; - break; - } - - if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + if( zRight && (pPragma->mPragFlag & PragFlag_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ @@ -102964,11 +107529,10 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; - sqlite3VdbeSetNumCols(v, 1); pParse->nMem = 1; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } } @@ -102977,11 +107541,12 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { + static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -102989,14 +107554,12 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - sqlite3VdbeSetNumCols(v, 3); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); - sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -103014,7 +107577,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "wal_autocheckpoint", + returnSingleInt(v, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -103024,8 +107587,9 @@ SQLITE_PRIVATE void sqlite3Pragma( /* ** PRAGMA shrink_memory ** - ** This pragma attempts to free as much memory as possible from the - ** current database connection. + ** IMPLEMENTATION-OF: R-23445-46109 This pragma causes the database + ** connection on which it is invoked to free up as much memory as it + ** can, by calling sqlite3_db_release_memory(). */ case PragTyp_SHRINK_MEMORY: { sqlite3_db_release_memory(db); @@ -103042,11 +107606,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** disables the timeout. */ /*case PragTyp_BUSY_TIMEOUT*/ default: { - assert( aPragmaNames[mid].ePragTyp==PragTyp_BUSY_TIMEOUT ); + assert( pPragma->ePragTyp==PragTyp_BUSY_TIMEOUT ); if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "timeout", db->busyTimeout); + returnSingleInt(v, "timeout", db->busyTimeout); break; } @@ -103054,15 +107618,19 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA soft_heap_limit ** PRAGMA soft_heap_limit = N ** - ** Call sqlite3_soft_heap_limit64(N). Return the result. If N is omitted, - ** use -1. + ** IMPLEMENTATION-OF: R-26343-45930 This pragma invokes the + ** sqlite3_soft_heap_limit64() interface with the argument N, if N is + ** specified and is a non-negative integer. + ** IMPLEMENTATION-OF: R-64451-07163 The soft_heap_limit pragma always + ** returns the same integer that would be returned by the + ** sqlite3_soft_heap_limit64(-1) C-language function. */ case PragTyp_SOFT_HEAP_LIMIT: { sqlite3_int64 N; if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); break; } @@ -103081,7 +107649,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(pParse, "threads", + returnSingleInt(v, "threads", sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -103094,17 +107662,15 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; + static const char *azCol[] = { "database", "status" }; int i; - sqlite3VdbeSetNumCols(v, 2); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; @@ -103112,7 +107678,7 @@ SQLITE_PRIVATE void sqlite3Pragma( SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -103188,6 +107754,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** interface, and routines that contribute to loading the database schema ** from disk. */ +/* #include "sqliteInt.h" */ /* ** Fill the InitData structure with an error message that indicates @@ -103200,13 +107767,13 @@ static void corruptSchema( ){ sqlite3 *db = pData->db; if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + char *z; if( zObj==0 ) zObj = "?"; - sqlite3SetString(pData->pzErrMsg, db, - "malformed database schema (%s)", zObj); - if( zExtra ){ - *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, - "%s - %s", *pData->pzErrMsg, zExtra); - } + z = sqlite3_mprintf("malformed database schema (%s)", zObj); + if( z && zExtra ) z = sqlite3_mprintf("%z - %s", z, zExtra); + sqlite3DbFree(db, *pData->pzErrMsg); + *pData->pzErrMsg = z; + if( z==0 ) db->mallocFailed = 1; } pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; } @@ -103241,7 +107808,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ if( argv[1]==0 ){ corruptSchema(pData, argv[0], 0); - }else if( argv[2] && argv[2][0] ){ + }else if( sqlite3_strnicmp(argv[2],"create ",7)==0 ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data @@ -103272,8 +107839,8 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char } } sqlite3_finalize(pStmt); - }else if( argv[0]==0 ){ - corruptSchema(pData, 0, 0); + }else if( argv[0]==0 || (argv[2]!=0 && argv[2][0]!=0) ){ + corruptSchema(pData, argv[0], 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -103398,7 +107965,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); if( rc!=SQLITE_OK ){ - sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; } openedTransaction = 1; @@ -103568,9 +108135,11 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; + ENC(db) = SCHEMA_ENC(db); for(i=0; rc==SQLITE_OK && inDb; i++){ if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; rc = sqlite3InitOne(db, i, pzErrMsg); @@ -103883,9 +108452,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -103946,7 +108518,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int sqlite3_prepare( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -103958,7 +108530,7 @@ SQLITE_API int sqlite3_prepare( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int sqlite3_prepare_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -103992,9 +108564,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ @@ -104032,7 +108606,7 @@ static int sqlite3Prepare16( ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int sqlite3_prepare16( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -104044,7 +108618,7 @@ SQLITE_API int sqlite3_prepare16( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int sqlite3_prepare16_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -104075,6 +108649,7 @@ SQLITE_API int sqlite3_prepare16_v2( ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Trace output macros @@ -104083,7 +108658,8 @@ SQLITE_API int sqlite3_prepare16_v2( /***/ int sqlite3SelectTrace = 0; # define SELECTTRACE(K,P,S,X) \ if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",\ + (S)->zSelName,(S)),\ sqlite3DebugPrintf X #else # define SELECTTRACE(K,P,S,X) @@ -104120,20 +108696,25 @@ struct SortCtx { #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* -** Delete all the content of a Select structure but do not deallocate -** the select structure itself. +** Delete all the content of a Select structure. Deallocate the structure +** itself only if bFree is true. */ -static void clearSelect(sqlite3 *db, Select *p){ - sqlite3ExprListDelete(db, p->pEList); - sqlite3SrcListDelete(db, p->pSrc); - sqlite3ExprDelete(db, p->pWhere); - sqlite3ExprListDelete(db, p->pGroupBy); - sqlite3ExprDelete(db, p->pHaving); - sqlite3ExprListDelete(db, p->pOrderBy); - sqlite3SelectDelete(db, p->pPrior); - sqlite3ExprDelete(db, p->pLimit); - sqlite3ExprDelete(db, p->pOffset); - sqlite3WithDelete(db, p->pWith); +static void clearSelect(sqlite3 *db, Select *p, int bFree){ + while( p ){ + Select *pPrior = p->pPrior; + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pLimit); + sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); + if( bFree ) sqlite3DbFree(db, p); + p = pPrior; + bFree = 1; + } } /* @@ -104168,7 +108749,6 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( Select standin; sqlite3 *db = pParse->db; pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); - assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ if( pNew==0 ){ assert( db->mallocFailed ); pNew = &standin; @@ -104188,12 +108768,11 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; - assert( pOffset==0 || pLimit!=0 ); + assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; if( db->mallocFailed ) { - clearSelect(db, pNew); - if( pNew!=&standin ) sqlite3DbFree(db, pNew); + clearSelect(db, pNew, pNew!=&standin); pNew = 0; }else{ assert( pNew->pSrc!=0 || pParse->nErr>0 ); @@ -104218,10 +108797,7 @@ SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ ** Delete the given Select structure and all of its substructures. */ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ - if( p ){ - clearSelect(db, p); - sqlite3DbFree(db, p); - } + clearSelect(db, p, 1); } /* @@ -104427,6 +109003,12 @@ static void setJoinExpr(Expr *p, int iTable){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->iRightJoinTable = (i16)iTable; + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + setJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } setJoinExpr(p->pLeft, iTable); p = p->pRight; } @@ -104461,12 +109043,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->jointype & JT_NATURAL ){ + if( pRight->fg.jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -104551,6 +109133,7 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ int nData, /* Number of elements in the data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ @@ -104564,6 +109147,7 @@ static void pushOntoSorter( int op; /* Opcode to add sorter record to sorter */ assert( bSeq==0 || bSeq==1 ); + assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -104571,7 +109155,8 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } @@ -104604,7 +109189,9 @@ static void pushOntoSorter( pKI = pOp->p4.pKeyInfo; memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); - pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + testcase( pKI->nXField>2 ); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, + pKI->nXField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(v); @@ -104622,20 +109209,17 @@ static void pushOntoSorter( } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( pSelect->iLimit ){ - int addr1, addr2; + int addr; int iLimit; if( pSelect->iOffset ){ iLimit = pSelect->iOffset+1; }else{ iLimit = pSelect->iLimit; } - addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); - addr2 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, addr1); + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); - sqlite3VdbeJumpHere(v, addr2); + sqlite3VdbeJumpHere(v, addr); } } @@ -104648,11 +109232,8 @@ static void codeOffset( int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); } } @@ -104776,8 +109357,13 @@ static void selectInnerLoop( /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - sqlite3ExprCodeExprList(pParse, pEList, regResult, - (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); + u8 ecelFlags; + if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ + ecelFlags = SQLITE_ECEL_DUP; + }else{ + ecelFlags = 0; + } + sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -104832,7 +109418,8 @@ static void selectInnerLoop( default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, + regResult); break; } } @@ -104874,6 +109461,8 @@ static void selectInnerLoop( int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); + testcase( eDest==SRT_Fifo ); + testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ @@ -104883,13 +109472,14 @@ static void selectInnerLoop( ** current row to the index and proceed with writing it to the ** output table as well. */ int addr = sqlite3VdbeCurrentAddr(v) + 4; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); assert( pSort==0 ); } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -104915,7 +109505,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -104941,7 +109531,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -104955,7 +109545,8 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -105032,7 +109623,7 @@ static void selectInnerLoop( ** the output for us. */ if( pSort==0 && p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); } } @@ -105115,7 +109706,7 @@ static KeyInfo *keyInfoFromExprList( int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; ilabelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; @@ -105289,10 +109878,7 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); } switch( eDest ){ - case SRT_Table: case SRT_EphemTab: { - testcase( eDest==SRT_Table ); - testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -105369,28 +109955,27 @@ static void generateSortTail( */ #ifdef SQLITE_ENABLE_COLUMN_METADATA # define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +#endif static const char *columnTypeImpl( NameContext *pNC, Expr *pExpr, +#ifdef SQLITE_ENABLE_COLUMN_METADATA const char **pzOrigDb, const char **pzOrigTab, const char **pzOrigCol, +#endif u8 *pEstWidth ){ - char const *zOrigDb = 0; - char const *zOrigTab = 0; - char const *zOrigCol = 0; -#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) -static const char *columnTypeImpl( - NameContext *pNC, - Expr *pExpr, - u8 *pEstWidth -){ -#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ char const *zType = 0; int j; u8 estWidth = 1; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#endif if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; switch( pExpr->op ){ @@ -105447,6 +110032,9 @@ static const char *columnTypeImpl( /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. + ** + ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been + ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; @@ -105637,7 +110225,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int selectColumnsFromExprList( +SQLITE_PRIVATE int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -105763,12 +110351,15 @@ static void selectAddColumnTypeAndCollation( a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ p = a[i].pExpr; - pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + if( pCol->zType==0 ){ + pCol->zType = sqlite3DbStrDup(db, + columnType(&sNC, p,0,0,0, &pCol->szEst)); + } szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl ){ + if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } @@ -105801,7 +110392,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ @@ -105858,7 +110449,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, n; + int n; if( p->iLimit ) return; /* @@ -105877,7 +110468,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeGoto(v, iBreak); }else if( n>=0 && p->nSelectRow>(u64)n ){ p->nSelectRow = n; } @@ -105885,7 +110476,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pLimit, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); } if( p->pOffset ){ p->iOffset = iOffset = ++pParse->nMem; @@ -105893,14 +110484,10 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0); sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1); } } } @@ -105922,7 +110509,10 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ pRet = 0; } assert( iCol>=0 ); - if( pRet==0 && iColpEList->nExpr ){ + /* iCol must be less than p->pEList->nExpr. Otherwise an error would + ** have been thrown during name resolution and we would not have gotten + ** this far */ + if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); } return pRet; @@ -105977,7 +110567,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -106042,7 +110632,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ + if( pSrc->a[i].fg.isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -106104,7 +110694,7 @@ static void generateWithRecursiveQuery( selectInnerLoop(pParse, p, p->pEList, iCurrent, 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak); VdbeCoverage(v); } sqlite3VdbeResolveLabel(v, addrCont); @@ -106112,13 +110702,17 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - p->pPrior = 0; - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; + if( p->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + }else{ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + } /* Keep running the loop until the Queue is empty */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeGoto(v, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: @@ -106137,6 +110731,48 @@ static int multiSelectOrderBy( SelectDest *pDest /* What to do with query results */ ); +/* +** Handle the special case of a compound-select that originates from a +** VALUES clause. By handling this as a special case, we avoid deep +** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT +** on a VALUES clause. +** +** Because the Select object originates from a VALUES clause: +** (1) It has no LIMIT or OFFSET +** (2) All terms are UNION ALL +** (3) There is no ORDER BY clause +*/ +static int multiSelectValues( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + Select *pPrior; + int nRow = 1; + int rc = 0; + assert( p->selFlags & SF_MultiValue ); + do{ + assert( p->selFlags & SF_Values ); + assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); + assert( p->pLimit==0 ); + assert( p->pOffset==0 ); + assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); + if( p->pPrior==0 ) break; + assert( p->pPrior->pNext==p ); + p = p->pPrior; + nRow++; + }while(1); + while( p ){ + pPrior = p->pPrior; + p->pPrior = 0; + rc = sqlite3Select(pParse, p, pDest); + p->pPrior = pPrior; + if( rc ) break; + p->nSelectRow = nRow; + p = p->pNext; + } + return rc; +} /* ** This routine is called to process a compound query form from @@ -106218,20 +110854,18 @@ static int multiSelect( dest.eDest = SRT_Table; } + /* Special handling for a compound-select that originates as a VALUES clause. + */ + if( p->selFlags & SF_MultiValue ){ + rc = multiSelectValues(pParse, p, &dest); + goto multi_select_end; + } + /* Make sure all SELECTs in the statement have the same number of elements ** in their result sets. */ assert( p->pEList && pPrior->pEList ); - if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - if( p->selFlags & SF_Values ){ - sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); - }else{ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - } - rc = 1; - goto multi_select_end; - } + assert( p->pEList->nExpr==pPrior->pEList->nExpr ); #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ @@ -106267,8 +110901,13 @@ static int multiSelect( p->iLimit = pPrior->iLimit; p->iOffset = pPrior->iOffset; if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeCoverage(v); + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0); + sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1); + } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); @@ -106523,6 +111162,19 @@ static int multiSelect( } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ +/* +** Error message for when two or more terms of a compound select have different +** size result sets. +*/ +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } +} + /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. @@ -106563,12 +111215,12 @@ static int generateOutputSubroutine( /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } @@ -106578,15 +111230,14 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ - case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - testcase( pDest->eDest==SRT_Table ); - testcase( pDest->eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); @@ -106603,7 +111254,7 @@ static int generateOutputSubroutine( */ case SRT_Set: { int r1; - assert( pIn->nSdst==1 ); + assert( pIn->nSdst==1 || pParse->nErr>0 ); pDest->affSdst = sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); @@ -106614,22 +111265,12 @@ static int generateOutputSubroutine( break; } -#if 0 /* Never occurs on an ORDER BY query */ - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } -#endif - /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. */ case SRT_Mem: { - assert( pIn->nSdst==1 ); + assert( pIn->nSdst==1 || pParse->nErr>0 ); testcase( pIn->nSdst!=1 ); sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); /* The LIMIT clause will jump out of the loop for us */ break; @@ -106644,7 +111285,7 @@ static int generateOutputSubroutine( pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); pDest->nSdst = pIn->nSdst; } - sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pDest->nSdst); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst); sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); break; } @@ -106668,7 +111309,7 @@ static int generateOutputSubroutine( /* Jump to the end of the loop if the LIMIT is reached. */ if( p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); } /* Generate the subroutine return @@ -106796,7 +111437,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -106860,8 +111501,8 @@ static int multiSelectOrderBy( if( aPermute ){ struct ExprList_item *pItem; for(i=0, pItem=pOrderBy->a; iu.x.iOrderByCol>0 - && pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + assert( pItem->u.x.iOrderByCol>0 ); + assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; } pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); @@ -106932,19 +111573,19 @@ static int multiSelectOrderBy( ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; @@ -106985,7 +111626,7 @@ static int multiSelectOrderBy( addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + sqlite3VdbeGoto(v, addrEofA); p->nSelectRow += pPrior->nSelectRow; } @@ -106999,7 +111640,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of AB @@ -107031,11 +111672,11 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); @@ -107071,14 +111712,14 @@ static int multiSelectOrderBy( /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ explainComposite(pParse, p->op, iSub1, iSub2, 0); - return SQLITE_OK; + return pParse->nErr!=0; } #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -107115,7 +111756,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -107138,25 +111779,28 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -107191,7 +111835,10 @@ static void substSelect( ** ** (1) The subquery and the outer query do not both use aggregates. ** -** (2) The subquery is not an aggregate or the outer query is not a join. +** (2) The subquery is not an aggregate or (2a) the outer query is not a join +** and (2b) the outer query does not use subqueries other than the one +** FROM-clause subquery that is a candidate for flattening. (2b is +** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) ** ** (3) The subquery is not the right operand of a left outer join ** (Originally ticket #306. Strengthened by ticket #3300) @@ -107217,8 +111864,8 @@ static void substSelect( ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we ** accidently carried the comment forward until 2014-09-15. Original -** text: "The subquery does not use aggregates or the outer query does not -** use LIMIT." +** text: "The subquery does not use aggregates or the outer query +** does not use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -107305,7 +111952,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -107328,8 +111975,17 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ - if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ + if( subqueryIsAgg ){ + if( isAgg ) return 0; /* Restriction (1) */ + if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ + if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) + || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 + || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 + ){ + return 0; /* Restriction (2b) */ + } + } + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, @@ -107399,7 +112055,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ return 0; } @@ -107419,10 +112075,10 @@ static int flattenSubquery( testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); + assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) || pSub1->pSrc->nSrc<1 - || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } @@ -107570,7 +112226,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; + jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -107591,9 +112247,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -107610,7 +112266,7 @@ static int flattenSubquery( pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].jointype = jointype; + pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -107632,11 +112288,6 @@ static int flattenSubquery( pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th @@ -107656,27 +112307,20 @@ static int flattenSubquery( assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -107702,7 +112346,7 @@ static int flattenSubquery( #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ - sqlite3DebugPrintf("After flattening:\n"); + SELECTTRACE(0x100,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -107711,6 +112355,77 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Make copies of relevant WHERE clause terms of the outer query into +** the WHERE clause of subquery. Example: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; +** +** Transformed into: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) +** WHERE x=5 AND y=10; +** +** The hope is that the terms added to the inner query will make it more +** efficient. +** +** Do not attempt this optimization if: +** +** (1) The inner query is an aggregate. (In that case, we'd really want +** to copy the outer WHERE-clause terms onto the HAVING clause of the +** inner query. But they probably won't help there so do not bother.) +** +** (2) The inner query is the recursive part of a common table expression. +** +** (3) The inner query has a LIMIT clause (since the changes to the WHERE +** close would change the meaning of the LIMIT). +** +** (4) The inner query is the right operand of a LEFT JOIN. (The caller +** enforces this restriction since this routine does not have enough +** information to know.) +** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN. +** +** Return 0 if no changes are made and non-zero if one or more WHERE clause +** terms are duplicated into the subquery. +*/ +static int pushDownWhereTerms( + sqlite3 *db, /* The database connection (for malloc()) */ + Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ + Expr *pWhere, /* The WHERE clause of the outer query */ + int iCursor /* Cursor number of the subquery */ +){ + Expr *pNew; + int nChng = 0; + if( pWhere==0 ) return 0; + if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + return 0; /* restrictions (1) and (2) */ + } + if( pSubq->pLimit!=0 ){ + return 0; /* restriction (3) */ + } + while( pWhere->op==TK_AND ){ + nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); + pWhere = pWhere->pLeft; + } + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ + if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + nChng++; + while( pSubq ){ + pNew = sqlite3ExprDup(db, pWhere, 0); + pNew = substExpr(db, pNew, iCursor, pSubq->pEList); + pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); + pSubq = pSubq->pPrior; + } + } + return nChng; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + /* ** Based on the contents of the AggInfo structure indicated by the first ** argument, this function checks if the following are true: @@ -107794,20 +112509,20 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndex ){ + if( pFrom->pTab && pFrom->fg.isIndexedBy ){ Table *pTab = pFrom->pTab; - char *zIndex = pFrom->zIndex; + char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); pIdx=pIdx->pNext ); if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + pFrom->pIBIndex = pIdx; } return SQLITE_OK; } @@ -107871,7 +112586,10 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ pNew->pOrderBy = 0; p->pPrior = 0; p->pNext = 0; + p->pWith = 0; p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; @@ -107965,12 +112683,12 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zErr is non-NULL at this point, then this is an illegal + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zErr ){ - sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); return SQLITE_ERROR; } @@ -107981,7 +112699,7 @@ static int withExpand( pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; assert( pFrom->pSelect ); @@ -107999,7 +112717,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->isRecursive = 1; + pItem->fg.isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -108015,7 +112733,7 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference: %s"; + pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); @@ -108023,7 +112741,7 @@ static int withExpand( for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; if( pCte->pCols ){ - if( pEList->nExpr!=pCte->pCols->nExpr ){ + if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){ sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", pCte->zName, pEList->nExpr, pCte->pCols->nExpr ); @@ -108033,16 +112751,16 @@ static int withExpand( pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references: %s"; + pCte->zCteErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference in a subquery: %s"; + pCte->zCteErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zErr = 0; + pCte->zCteErr = 0; pParse->pWith = pSavedWith; } @@ -108114,7 +112832,9 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( pWalker->xSelectCallback2==selectPopWith ){ + sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + } /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -108127,17 +112847,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->isRecursive==0 || pFrom->pTab ); - if( pFrom->isRecursive ) continue; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); -#ifndef SQLITE_OMIT_CTE - selectPopWith(pWalker, p); -#endif - return WRC_Prune; - } + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->fg.isRecursive ) continue; + assert( pFrom->pTab==0 ); #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else @@ -108148,13 +112860,13 @@ static int selectExpander(Walker *pWalker, Select *p){ /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); assert( pFrom->pTab==0 ); - sqlite3WalkSelect(pWalker, pSel); + if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; @@ -108173,12 +112885,19 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); + if( pFrom->fg.isTabFunc && !IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pTab->zName); + return WRC_Abort; + } pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } @@ -108224,13 +112943,6 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; - /* When processing FROM-clause subqueries, it is always the case - ** that full_column_names=OFF and short_column_names=ON. The - ** sqlite3ResultSetOfSelect() routine makes it so. */ - assert( (p->selFlags & SF_NestedFrom)==0 - || ((flags & SQLITE_FullColNames)==0 && - (flags & SQLITE_ShortColNames)!=0) ); - for(k=0; knExpr; k++){ pE = a[k].pExpr; pRight = pE->pRight; @@ -108298,7 +113010,7 @@ static int selectExpander(Walker *pWalker, Select *p){ tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 + if( (pFrom->fg.jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -108405,7 +113117,9 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - w.xSelectCallback2 = selectPopWith; + if( (pSelect->selFlags & SF_MultiValue)==0 ){ + w.xSelectCallback2 = selectPopWith; + } sqlite3WalkSelect(&w, pSelect); } @@ -108431,19 +113145,19 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + assert( (p->selFlags & SF_HasTypeInfo)==0 ); + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -108582,14 +113296,15 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; } if( pF->iDistinct>=0 ){ addrNext = sqlite3VdbeMakeLabel(v); - assert( nArg==1 ); + testcase( nArg==0 ); /* Error condition */ + testcase( nArg>1 ); /* Also an error */ codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ @@ -108606,7 +113321,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); @@ -108689,7 +113404,7 @@ SQLITE_PRIVATE int sqlite3Select( WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ Vdbe *v; /* The virtual machine under construction */ int isAgg; /* True for select lists like "count(*)" */ - ExprList *pEList; /* List of columns to extract. */ + ExprList *pEList = 0; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ @@ -108739,36 +113454,90 @@ SQLITE_PRIVATE int sqlite3Select( memset(&sSort, 0, sizeof(sSort)); sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; - pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ goto select_end; } + assert( p->pEList!=0 ); isAgg = (p->selFlags & SF_Aggregate)!=0; - assert( pEList!=0 ); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif - /* Begin generating code. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto select_end; /* If writing to memory or generating a set ** only a single column may be output. */ #ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + if( checkForMultiColumnSelectError(pParse, pDest, p->pEList->nExpr) ){ goto select_end; } #endif - /* Generate code for all sub-queries in the FROM clause + /* Try to flatten subqueries in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; - SelectDest dest; Select *pSub = pItem->pSelect; int isAggSub; + Table *pTab = pItem->pTab; + if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; + } + i = -1; + } + pTabList = p->pSrc; + if( db->mallocFailed ) goto select_end; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } +#endif + + /* Get a pointer the VDBE under construction, allocating a new VDBE if one + ** does not already exist */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto select_end; + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* Handle compound SELECT statements using the separate multiSelect() + ** procedure. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif + return rc; + } +#endif + + /* Generate code for all sub-queries in the FROM clause + */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; if( pSub==0 ) continue; /* Sometimes the code for a subquery will be generated more than @@ -108778,7 +113547,7 @@ SQLITE_PRIVATE int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ + if( pItem->fg.viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -108793,16 +113562,25 @@ SQLITE_PRIVATE int sqlite3Select( */ pParse->nHeight += sqlite3SelectExprHeight(p); - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ - /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( (pItem->fg.jointype & JT_OUTER)==0 + && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + sqlite3TreeViewSelect(0, p, 0); } - i = -1; - }else if( pTabList->nSrc==1 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) +#endif + } + + /* Generate code to implement the subquery + */ + if( pTabList->nSrc==1 + && (p->selFlags & SF_All)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -108816,7 +113594,7 @@ SQLITE_PRIVATE int sqlite3Select( explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; + pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); @@ -108834,7 +113612,7 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ + if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ @@ -108853,33 +113631,23 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeChangeP1(v, topAddr, retAddr); sqlite3ClearTempRegCache(pParse); } - if( /*pParse->nErr ||*/ db->mallocFailed ){ - goto select_end; - } + if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); - pTabList = p->pSrc; - if( !IgnorableOrderby(pDest) ){ - sSort.pOrderBy = p->pOrderBy; - } } - pEList = p->pEList; #endif + + /* Various elements of the SELECT copied into local variables for + ** convenience */ + pEList = p->pEList; pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* If there is are a sequence of queries, do the earlier ones first. - */ - if( p->pPrior ){ - rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); #if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p,("end compound-select processing\n")); - pParse->nSelectIndent--; -#endif - return rc; + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); } #endif @@ -108891,7 +113659,7 @@ SQLITE_PRIVATE int sqlite3Select( ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally @@ -108899,28 +113667,27 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; + pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); } - /* If there is an ORDER BY clause, then this sorting - ** index might end up being unused if the data can be - ** extracted in pre-sorted order. If that is the case, then the - ** OP_OpenEphemeral instruction will be changed to an OP_Noop once - ** we figure out that the sorting index is not needed. The addrSortIndex - ** variable is used to facilitate that change. + /* If there is an ORDER BY clause, then create an ephemeral index to + ** do the sorting. But this sorting ephemeral index might end up + ** being unused if the data can be extracted in pre-sorted order. + ** If that is the case, then the OP_OpenEphemeral instruction will be + ** changed to an OP_Noop once we figure out that the sorting index is + ** not needed. The sSort.addrSortIndex variable is used to facilitate + ** that change. */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, pEList->nExpr); sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -108943,18 +113710,18 @@ SQLITE_PRIVATE int sqlite3Select( p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open a virtual index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -109032,11 +113799,10 @@ SQLITE_PRIVATE int sqlite3Select( p->nSelectRow = 1; } - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then it may be possible to disable the ORDER BY clause ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY may use a + ** in the correct order. It also may not - the GROUP BY might use a ** database index that causes rows to be grouped together as required ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp @@ -109079,7 +113845,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -109094,7 +113860,7 @@ SQLITE_PRIVATE int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, sAggInfo.nColumn); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -109160,7 +113926,7 @@ SQLITE_PRIVATE int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; inExpr; j++){ if( groupBySort ){ @@ -109226,8 +113993,8 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -109249,7 +114016,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -109271,7 +114038,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Jump over the subroutines */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + sqlite3VdbeGoto(v, addrEnd); /* Generate a subroutine that outputs a single row of the result ** set. This subroutine first looks at the iUseFlag. If iUseFlag @@ -109286,7 +114053,8 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); @@ -109424,7 +114192,7 @@ SQLITE_PRIVATE int sqlite3Select( updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); + sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -109450,7 +114218,8 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + explainTempTable(pParse, + sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -109458,10 +114227,9 @@ SQLITE_PRIVATE int sqlite3Select( */ sqlite3VdbeResolveLabel(v, iEnd); - /* The SELECT was successfully coded. Set the return code to 0 - ** to indicate no errors. - */ - rc = 0; + /* The SELECT has been coded. If there is an error in the Parse structure, + ** set the return code to 1. Otherwise 0. */ + rc = (pParse->nErr>0); /* Control jumps to here if an error is encountered above, or upon ** successful coding of the SELECT. @@ -109484,100 +114252,6 @@ SQLITE_PRIVATE int sqlite3Select( return rc; } -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable description of a the Select object. -*/ -SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ - int n = 0; - pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewLine(pView, "SELECT%s%s", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") - ); - if( p->pSrc && p->pSrc->nSrc ) n++; - if( p->pWhere ) n++; - if( p->pGroupBy ) n++; - if( p->pHaving ) n++; - if( p->pOrderBy ) n++; - if( p->pLimit ) n++; - if( p->pOffset ) n++; - if( p->pPrior ) n++; - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, 0, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); - } - if( pItem->jointype & JT_LEFT ){ - sqlite3XPrintf(&x, 0, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); - } - if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", (n--)>0); - sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); - } - if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); - } - if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", (n--)>0); - sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); - } - if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); - } - if( p->pPrior ){ - const char *zOp = "UNION"; - switch( p->op ){ - case TK_ALL: zOp = "UNION ALL"; break; - case TK_INTERSECT: zOp = "INTERSECT"; break; - case TK_EXCEPT: zOp = "EXCEPT"; break; - } - sqlite3TreeViewItem(pView, zOp, (n--)>0); - sqlite3TreeViewSelect(pView, p->pPrior, 0); - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - /************** End of select.c **********************************************/ /************** Begin file table.c *******************************************/ /* @@ -109598,6 +114272,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m ** These routines are in a separate files so that they will not be linked ** if they are not used. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -109671,7 +114346,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ z = 0; }else{ int n = sqlite3Strlen30(argv[i])+1; - z = sqlite3_malloc( n ); + z = sqlite3_malloc64( n ); if( z==0 ) goto malloc_failed; memcpy(z, argv[i], n); } @@ -109696,7 +114371,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ ** Instead, the entire table should be passed to sqlite3_free_table() when ** the calling procedure is finished using it. */ -SQLITE_API int sqlite3_get_table( +SQLITE_API int SQLITE_STDCALL sqlite3_get_table( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ char ***pazResult, /* Write the result table here */ @@ -109707,6 +114382,9 @@ SQLITE_API int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; @@ -109717,7 +114395,7 @@ SQLITE_API int sqlite3_get_table( res.nData = 1; res.nAlloc = 20; res.rc = SQLITE_OK; - res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); + res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); if( res.azResult==0 ){ db->errCode = SQLITE_NOMEM; return SQLITE_NOMEM; @@ -109745,7 +114423,7 @@ SQLITE_API int sqlite3_get_table( } if( res.nAlloc>res.nData ){ char **azNew; - azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); + azNew = sqlite3_realloc64( res.azResult, sizeof(char*)*res.nData ); if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; @@ -109762,7 +114440,7 @@ SQLITE_API int sqlite3_get_table( /* ** This routine frees the space the sqlite3_get_table() malloced. */ -SQLITE_API void sqlite3_free_table( +SQLITE_API void SQLITE_STDCALL sqlite3_free_table( char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ @@ -109791,6 +114469,7 @@ SQLITE_API void sqlite3_free_table( ************************************************************************* ** This file contains the implementation for TRIGGERs */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_TRIGGER /* @@ -109973,7 +114652,6 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); - pParse->nErr++; goto trigger_cleanup; } @@ -110153,12 +114831,12 @@ static TriggerStep *triggerStepAllocate( ){ TriggerStep *pTriggerStep; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n); + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); if( pTriggerStep ){ char *z = (char*)&pTriggerStep[1]; memcpy(z, pName->z, pName->n); - pTriggerStep->target.z = z; - pTriggerStep->target.n = pName->n; + sqlite3Dequote(z); + pTriggerStep->zTarget = z; pTriggerStep->op = op; } return pTriggerStep; @@ -110441,7 +115119,7 @@ SQLITE_PRIVATE Trigger *sqlite3TriggersExist( } /* -** Convert the pStep->target token into a SrcList and return a pointer +** Convert the pStep->zTarget string into a SrcList and return a pointer ** to that SrcList. ** ** This routine adds a specific database name, if needed, to the target when @@ -110454,17 +115132,17 @@ static SrcList *targetSrcList( Parse *pParse, /* The parsing context */ TriggerStep *pStep /* The trigger containing the target token */ ){ + sqlite3 *db = pParse->db; int iDb; /* Index of the database to use */ SrcList *pSrc; /* SrcList to be returned */ - pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); + pSrc = sqlite3SrcListAppend(db, 0, 0, 0); if( pSrc ){ assert( pSrc->nSrc>0 ); - assert( pSrc->a!=0 ); - iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); + pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget); + iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema); if( iDb==0 || iDb>=2 ){ - sqlite3 *db = pParse->db; - assert( iDbdb->nDb ); + assert( iDbnDb ); pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); } } @@ -110576,6 +115254,7 @@ static void transferParseError(Parse *pTo, Parse *pFrom){ if( pTo->nErr==0 ){ pTo->zErrMsg = pFrom->zErrMsg; pTo->nErr = pFrom->nErr; + pTo->rc = pFrom->rc; }else{ sqlite3DbFree(pFrom->db, pFrom->zErrMsg); } @@ -110914,6 +115593,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Forward declaration */ @@ -111035,9 +115715,9 @@ SQLITE_PRIVATE void sqlite3Update( /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; /* Content of the NEW.* table in triggers */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ @@ -111173,7 +115853,9 @@ SQLITE_PRIVATE void sqlite3Update( /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. + ** the key for accessing each index. + ** + ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -111182,7 +115864,8 @@ SQLITE_PRIVATE void sqlite3Update( }else{ reg = 0; for(i=0; inKeyCol; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ + i16 iIdxCol = pIdx->aiColumn[i]; + if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; break; } @@ -111198,29 +115881,20 @@ SQLITE_PRIVATE void sqlite3Update( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - pWhere = 0; - pTabList = 0; - goto update_cleanup; - } -#endif - /* Allocate required registers. */ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; + if( !IsVirtual(pTab) ){ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; pParse->nMem += pTab->nCol; } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; /* Start the view context. */ if( isView ){ @@ -111243,6 +115917,15 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif + /* Begin the database scan */ if( HasRowid(pTab) ){ @@ -111282,6 +115965,7 @@ SQLITE_PRIVATE void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } @@ -111291,7 +115975,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -111404,7 +116088,6 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -111462,7 +116145,7 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int j1 = 0; /* Address of jump instruction */ + int addr1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ @@ -111478,20 +116161,20 @@ SQLITE_PRIVATE void sqlite3Update( /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } if( hasFK ){ @@ -111528,7 +116211,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + sqlite3VdbeGoto(v, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); @@ -111582,21 +116265,23 @@ SQLITE_PRIVATE void sqlite3Update( /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemeral table that contains +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) +** (B) The revised rowid for the row. ** (C) The content of every column in the row. ** -** Then we loop over this ephemeral table and for each row in -** the ephemeral table call VUpdate. -** -** When finished, drop the ephemeral table. +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. ** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ @@ -111609,68 +116294,96 @@ static void updateVirtualTable( int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; - - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); + WhereInfo *pWInfo; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int bOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ + + /* Allocate nArg registers to martial the arguments to VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ + assert( v ); + ephemTab = pParse->nTab++; + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); } - assert( pTab->iPKey<0 ); for(i=0; inCol; i++){ if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ - assert( v ); - ephemTab = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); - sqlite3Select(pParse, pSelect, &dest); + bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); - for(i=0; inCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + if( bOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. Also, if this is a top-level parse (not a trigger), clear the + ** multi-write flag so that the VM does not open a statement journal */ + sqlite3VdbeChangeToNoop(v, addr); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + + + if( bOnePass==0 ){ + /* End the virtual table scan */ + sqlite3WhereEnd(pWInfo); + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; inCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + /* End of the ephemeral table scan. Or, if using the onepass strategy, + ** jump to here if the scan visited zero rows. */ + if( bOnePass==0 ){ + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + }else{ + sqlite3WhereEnd(pWInfo); + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -111692,6 +116405,8 @@ static void updateVirtualTable( ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) /* @@ -111770,7 +116485,7 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** overwriting the database with the vacuumed content. ** ** Only 1x temporary space and only 1x writes would be required if -** the copy of step (3) were replace by deleting the original database +** the copy of step (3) were replaced by deleting the original database ** and renaming the transient database as the original. But that will ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the @@ -111860,7 +116575,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** cause problems for the call to BtreeSetPageSize() below. */ sqlite3BtreeCommit(pTemp); - nRes = sqlite3BtreeGetReserve(pMain); + nRes = sqlite3BtreeGetOptimalReserve(pMain); /* A VACUUM cannot change the pagesize of an encrypted database. */ #ifdef SQLITE_HAS_CODEC @@ -111926,6 +116641,8 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ + assert( (db->flags & SQLITE_Vacuum)==0 ); + db->flags |= SQLITE_Vacuum; rc = execExecSql(db, pzErrMsg, "SELECT 'INSERT INTO vacuum_db.' || quote(name) " "|| ' SELECT * FROM main.' || quote(name) || ';'" @@ -111933,6 +116650,8 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ "WHERE type = 'table' AND name!='sqlite_sequence' " " AND coalesce(rootpage,1)>0" ); + assert( (db->flags & SQLITE_Vacuum)!=0 ); + db->flags &= ~SQLITE_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Copy over the sequence table @@ -112060,6 +116779,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** This file contains code used to help implement virtual tables. */ #ifndef SQLITE_OMIT_VIRTUALTABLE +/* #include "sqliteInt.h" */ /* ** Before a virtual table xCreate() or xConnect() method is invoked, the @@ -112071,6 +116791,8 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ struct VtabCtx { VTable *pVTable; /* The virtual table being constructed */ Table *pTab; /* The Table object to which the virtual table belongs */ + VtabCtx *pPrior; /* Parent context (if any) */ + int bDeclared; /* True after sqlite3_declare_vtab() is called */ }; /* @@ -112103,6 +116825,7 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ @@ -112122,25 +116845,31 @@ static int createModule( /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int sqlite3_create_module( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int sqlite3_create_module_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -112324,23 +117053,17 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + int nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ - int j; - for(j=0; jazModuleArg[j]); - } sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ + int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; + pTable->azModuleArg = azModuleArg; } - pTable->azModuleArg = azModuleArg; } /* @@ -112373,7 +117096,12 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. @@ -112425,6 +117153,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ char *zStmt; char *zWhere; int iDb; + int iReg; Vdbe *v; /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ @@ -112459,8 +117188,10 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); - sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, - pTab->zName, sqlite3Strlen30(pTab->zName) + 1); + + iReg = ++pParse->nMem; + sqlite3VdbeLoadString(v, iReg, pTab->zName); + sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); } /* If we are rereading the sqlite_master table create the in-memory @@ -112503,7 +117234,7 @@ SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ pArg->z = p->z; pArg->n = p->n; }else{ - assert(pArg->z < p->z); + assert(pArg->z <= p->z); pArg->n = (int)(&p->z[p->n] - pArg->z); } } @@ -112520,15 +117251,27 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx, *pPriorCtx; + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; int nArg = pTab->nModuleArg; char *zErr = 0; - char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + char *zModuleName; int iDb; + VtabCtx *pCtx; + + /* Check that the virtual-table is not already being initialized */ + for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){ + if( pCtx->pTab==pTab ){ + *pzErr = sqlite3MPrintf(db, + "vtable constructor called recursively: %s", pTab->zName + ); + return SQLITE_LOCKED; + } + } + zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); if( !zModuleName ){ return SQLITE_NOMEM; } @@ -112549,11 +117292,13 @@ static int vtabCallConstructor( assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; - pPriorCtx = db->pVtabCtx; + sCtx.pPrior = db->pVtabCtx; + sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = pPriorCtx; + db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ if( zErr==0 ){ @@ -112569,13 +117314,14 @@ static int vtabCallConstructor( memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( sCtx.pTab ){ + if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ int iCol; + u8 oooHidden = 0; /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". @@ -112588,7 +117334,10 @@ static int vtabCallConstructor( char *zType = pTab->aCol[iCol].zType; int nType; int i = 0; - if( !zType ) continue; + if( !zType ){ + pTab->tabFlags |= oooHidden; + continue; + } nType = sqlite3Strlen30(zType); if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ for(i=0; iaCol[iCol].colFlags |= COLFLAG_HIDDEN; + oooHidden = TF_OOOHidden; + }else{ + pTab->tabFlags |= oooHidden; } } } @@ -112714,7 +117466,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( !pMod ){ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -112738,19 +117490,26 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** valid to call this function from within the xCreate() or xConnect() of a ** virtual table module. */ -SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ +SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + VtabCtx *pCtx; Parse *pParse; - int rc = SQLITE_OK; Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); - if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ + pCtx = db->pVtabCtx; + if( !pCtx || pCtx->bDeclared ){ sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( (pTab->tabFlags & TF_Virtual)!=0 ); pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); @@ -112773,7 +117532,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } - db->pVtabCtx->pTab = 0; + pCtx->bDeclared = 1; }else{ sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); @@ -112808,11 +117567,18 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ - VTable *p = vtabDisconnectAll(db, pTab); - - assert( rc==SQLITE_OK ); - rc = p->pMod->pModule->xDestroy(p->pVtab); - + VTable *p; + int (*xDestroy)(sqlite3_vtab *); + for(p=pTab->pVTable; p; p=p->pNext){ + assert( p->pVtab ); + if( p->pVtab->nRef>0 ){ + return SQLITE_LOCKED; + } + } + p = vtabDisconnectAll(db, pTab); + xDestroy = p->pMod->pModule->xDestroy; + assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -112836,8 +117602,10 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab static void callFinaliser(sqlite3 *db, int offset){ int i; if( db->aVTrans ){ + VTable **aVTrans = db->aVTrans; + db->aVTrans = 0; for(i=0; inVTrans; i++){ - VTable *pVTab = db->aVTrans[i]; + VTable *pVTab = aVTrans[i]; sqlite3_vtab *p = pVTab->pVtab; if( p ){ int (*x)(sqlite3_vtab *); @@ -112847,9 +117615,8 @@ static void callFinaliser(sqlite3 *db, int offset){ pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } - sqlite3DbFree(db, db->aVTrans); + sqlite3DbFree(db, aVTrans); db->nVTrans = 0; - db->aVTrans = 0; } } @@ -112937,7 +117704,9 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); + if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1); } } } @@ -112963,7 +117732,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ int rc = SQLITE_OK; assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); - assert( iSavepoint>=0 ); + assert( iSavepoint>=-1 ); if( db->aVTrans ){ int i; for(i=0; rc==SQLITE_OK && inVTrans; i++){ @@ -113081,7 +117850,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ if( pTab==pToplevel->apVtabLock[i] ) return; } n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); + apVtabLock = sqlite3_realloc64(pToplevel->apVtabLock, n); if( apVtabLock ){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; @@ -113090,6 +117859,67 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } } +/* +** Check to see if virtual tale module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if the eponymous virtual table instance exists +** when this routine returns, and return zero if it does not exist. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int nName; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + nName = sqlite3Strlen30(pMod->zName) + 1; + pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName); + if( pTab==0 ) return 0; + pMod->pEpoTab = pTab; + pTab->zName = (char*)&pTab[1]; + memcpy(pTab->zName, pMod->zName, nName); + pTab->nRef = 1; + pTab->pSchema = db->aDb[0].pSchema; + pTab->tabFlags |= TF_Virtual; + pTab->nModuleArg = 0; + pTab->iPKey = -1; + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(db, pTab, 0); + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + return 0; + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( pTab!=0 ){ + sqlite3DeleteColumnNames(db, pTab); + sqlite3VtabClear(db, pTab); + sqlite3DbFree(db, pTab); + pMod->pEpoTab = 0; + } +} + /* ** Return the ON CONFLICT resolution mode in effect for the virtual ** table update operation currently in progress. @@ -113097,10 +117927,13 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ ** The results of this routine are undefined unless it is called from ** within an xUpdate method. */ -SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ +SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -113112,12 +117945,14 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ ** the SQLite core with additional information about the behavior ** of the virtual table being implemented. */ -SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ +SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { @@ -113144,9 +117979,9 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ /************** End of vtab.c ************************************************/ -/************** Begin file where.c *******************************************/ +/************** Begin file wherecode.c ***************************************/ /* -** 2001 September 15 +** 2015-06-06 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -113157,13 +117992,15 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** ************************************************************************* ** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. This module is responsible for -** generating the code that loops through a table looking for applicable -** rows. Indices are selected and used to speed the search when doing -** so is applicable. Because this module is responsible for selecting -** indices, you might also think of this module as the "query optimizer". +** the WHERE clause of SQL statements. +** +** This file was split off from where.c on 2015-06-06 in order to reduce the +** size of where.c and make it easier to edit. This file contains the routines +** that actually generate the bulk of the WHERE loop code. The original where.c +** file retains the code that does query planning and analysis. */ -/************** Include whereInt.h in the middle of where.c ******************/ +/* #include "sqliteInt.h" */ +/************** Include whereInt.h in the middle of wherecode.c **************/ /************** Begin file whereInt.h ****************************************/ /* ** 2013-11-12 @@ -113186,7 +118023,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace; #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) @@ -113236,6 +118073,8 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ + int iLikeRepCntr; /* LIKE range processing counter register */ + int addrLikeRep; /* LIKE range processing address */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ @@ -113252,6 +118091,9 @@ struct WhereLevel { } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif }; /* @@ -113282,7 +118124,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -113295,12 +118136,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a @@ -113323,10 +118165,6 @@ struct WhereOrSet { WhereOrCost a[N_OR_COST]; /* Set of best costs */ }; - -/* Forward declaration of methods */ -static int whereLoopResize(sqlite3*, WhereLoop*, int); - /* ** Each instance of this object holds a sequence of WhereLoop objects ** that implement some or all of a query plan. @@ -113417,7 +118255,7 @@ struct WhereTerm { } u; LogEst truthProb; /* Probability of truth for this expression */ u16 eOperator; /* A WO_xx value describing */ - u8 wtFlags; /* TERM_xxx bit flags. See below */ + u16 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ WhereClause *pWC; /* The clause this term is part of */ Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ @@ -113439,6 +118277,10 @@ struct WhereTerm { #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif +#define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ +#define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ +#define TERM_LIKE 0x400 /* The original LIKE operator */ +#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -113448,12 +118290,14 @@ struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; /* @@ -113530,6 +118374,11 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; +/* +** Initialize a WhereMaskSet object +*/ +#define initMaskSet(P) (P)->n=0 + /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -113567,7 +118416,7 @@ struct WhereInfo { u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 sorted; /* True if really sorted (not just grouped) */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ u8 nLevel; /* Number of nested loop */ @@ -113581,27 +118430,85 @@ struct WhereInfo { WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +/* +** Private interfaces - callable only by other where.c routines. +** +** where.c: +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +); + +/* wherecode.c: */ +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +); +#else +# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +); +#else +# define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) +#endif +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +); + +/* whereexpr.c: */ +SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); +SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); + + + + + /* ** Bitmasks for the operators on WhereTerm objects. These are all ** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for ** particular WhereTerms within a WhereClause. */ -#define WO_IN 0x001 -#define WO_EQ 0x002 +#define WO_IN 0x0001 +#define WO_EQ 0x0002 #define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) #define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x040 -#define WO_ISNULL 0x080 -#define WO_OR 0x100 /* Two or more OR-connected terms */ -#define WO_AND 0x200 /* Two or more AND-connected terms */ -#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ -#define WO_NOOP 0x800 /* This term does not restrict search space */ +#define WO_MATCH 0x0040 +#define WO_IS 0x0080 +#define WO_ISNULL 0x0100 +#define WO_OR 0x0200 /* Two or more OR-connected terms */ +#define WO_AND 0x0400 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ +#define WO_NOOP 0x1000 /* This term does not restrict search space */ -#define WO_ALL 0xfff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ +#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ /* ** These are definitions of bits in the WhereLoop.wsFlags field. @@ -113626,3798 +118533,4501 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ /************** End of whereInt.h ********************************************/ -/************** Continuing where we left off in where.c **********************/ - -/* -** Return the estimated number of output rows from a WHERE clause -*/ -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return sqlite3LogEstToInt(pWInfo->nRowOut); -} - -/* -** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this -** WHERE clause returns outputs for DISTINCT processing. -*/ -SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ - return pWInfo->eDistinct; -} - -/* -** Return TRUE if the WHERE clause returns rows in ORDER BY order. -** Return FALSE if the output needs to be sorted. -*/ -SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; -} +/************** Continuing where we left off in wherecode.c ******************/ +#ifndef SQLITE_OMIT_EXPLAIN /* -** Return the VDBE address or label to jump to in order to continue -** immediately with the next row of a WHERE clause. +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. */ -SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ - assert( pWInfo->iContinue!=0 ); - return pWInfo->iContinue; +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppendAll(pStr, zColumn); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); } /* -** Return the VDBE address or label to jump to in order to break -** out of a WHERE loop. +** Return the name of the i-th column of the pIdx index. */ -SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ - return pWInfo->iBreak; +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zName; } /* -** Return TRUE if an UPDATE or DELETE statement can operate directly on -** the rowids returned by a WHERE clause. Return FALSE if doing an -** UPDATE or DELETE might change subsequent WHERE clause results. +** Argument pLevel describes a strategy for scanning table pTab. This +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** -** If the ONEPASS optimization is used (if this routine returns true) -** then also write the indices of open cursors used by ONEPASS -** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data -** table and iaCur[1] gets the cursor used by an auxiliary index. -** Either value may be -1, indicating that cursor is not used. -** Any cursors returned will have been opened for writing. +** For example, if the query: ** -** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is -** unable to use the ONEPASS optimization. +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" */ -SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ - memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); - return pWInfo->okOnePass; -} +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->nSkip; + int i, j; -/* -** Move the content of pSrc into pDest -*/ -static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ - pDest->n = pSrc->n; - memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); -} + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); + for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); + } -/* -** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. -** -** The new entry might overwrite an existing entry, or it might be -** appended, or it might be discarded. Do whatever is the right thing -** so that pSet keeps the N_OR_COST best entries seen so far. -*/ -static int whereOrInsert( - WhereOrSet *pSet, /* The WhereOrSet to be updated */ - Bitmask prereq, /* Prerequisites of the new entry */ - LogEst rRun, /* Run-cost of the new entry */ - LogEst nOut /* Number of outputs for the new entry */ -){ - u16 i; - WhereOrCost *p; - for(i=pSet->n, p=pSet->a; i>0; i--, p++){ - if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ - goto whereOrInsert_done; - } - if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ - return 0; - } + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + const char *z = explainIndexColumnName(pIndex, i); + explainAppendTerm(pStr, i++, z, ">"); } - if( pSet->na[pSet->n++]; - p->nOut = nOut; - }else{ - p = pSet->a; - for(i=1; in; i++){ - if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; - } - if( p->rRun<=rRun ) return 0; + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + const char *z = explainIndexColumnName(pIndex, j); + explainAppendTerm(pStr, i, z, "<"); } -whereOrInsert_done: - p->prereq = prereq; - p->rRun = rRun; - if( p->nOut>nOut ) p->nOut = nOut; - return 1; + sqlite3StrAccumAppend(pStr, ")", 1); } /* -** Initialize a preallocated WhereClause structure. +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ -static void whereClauseInit( - WhereClause *pWC, /* The WhereClause to be initialized */ - WhereInfo *pWInfo /* The WHERE processing context */ +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ - pWC->pWInfo = pWInfo; - pWC->pOuter = 0; - pWC->nTerm = 0; - pWC->nSlot = ArraySize(pWC->aStatic); - pWC->a = pWC->aStatic; -} - -/* Forward reference */ -static void whereClauseClear(WhereClause*); + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pParse->explain==2 ) +#endif + { + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ -/* -** Deallocate all memory associated with a WhereOrInfo object. -*/ -static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; -/* -** Deallocate all memory associated with a WhereAndInfo object. -*/ -static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); -/* -** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of whereClauseInit(). -*/ -static void whereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; - sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); + if( pItem->pSelect ){ + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); + }else{ + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); + + if( pItem->zAlias ){ + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); } - } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } -} + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; -/* -** Add a single new WhereTerm entry to the WhereClause object pWC. -** The new WhereTerm object is constructed from Expr p and with wtFlags. -** The index in pWC->a[] of the new WhereTerm is returned on success. -** 0 is returned if the new WhereTerm could not be added due to a memory -** allocation error. The memory allocation failure will be recorded in -** the db->mallocFailed flag so that higher-level functions can detect it. -** -** This routine will increase the size of the pWC->a[] array as necessary. -** -** If the wtFlags argument includes TERM_DYNAMIC, then responsibility -** for freeing the expression p is assumed by the WhereClause object pWC. -** This is true even if this routine fails to allocate a new WhereTerm. -** -** WARNING: This routine might reallocate the space used to store -** WhereTerms. All pointers to WhereTerms should be invalidated after -** calling this routine. Such pointers may be reinitialized by referencing -** the pWC->a[] array. -*/ -static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ - WhereTerm *pTerm; - int idx; - testcase( wtFlags & TERM_VIRTUAL ); - if( pWC->nTerm>=pWC->nSlot ){ - WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); - if( pWC->a==0 ){ - if( wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, p); + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; } - pWC->a = pOld; - return 0; + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRangeOp; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRangeOp = "="; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRangeOp = ">? AND rowid<"; + }else if( flags&WHERE_BTM_LIMIT ){ + zRangeOp = ">"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRangeOp = "<"; + } + sqlite3XPrintf(&str, 0, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); } - memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ + sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); - } - pTerm = &pWC->a[idx = pWC->nTerm++]; - if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; - }else{ - pTerm->truthProb = 1; +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } - pTerm->pExpr = sqlite3ExprSkipCollate(p); - pTerm->wtFlags = wtFlags; - pTerm->pWC = pWC; - pTerm->iParent = -1; - return idx; + return ret; } +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** This routine identifies subexpressions in the WHERE clause where -** each subexpression is separated by the AND operator or some other -** operator specified in the op parameter. The WhereClause structure -** is filled with pointers to subexpressions. For example: -** -** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) -** \________/ \_______________/ \________________/ -** slot[0] slot[1] slot[2] +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. ** -** The original WHERE clause in pExpr is unaltered. All this routine -** does is make slot[] entries point to substructure within pExpr. -** -** In the previous sentence and in the diagram, "slot[]" refers to -** the WhereClause.a[] array. The slot[] array grows as needed to contain -** all terms of the WHERE clause. +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. */ -static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - pWC->op = op; - if( pExpr==0 ) return; - if( pExpr->op!=op ){ - whereClauseInsert(pWC, pExpr, 0); +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; }else{ - whereSplit(pWC, pExpr->pLeft, op); - whereSplit(pWC, pExpr->pRight, op); + zObj = pSrclist->a[pLvl->iFrom].zName; } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); } +#endif -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 /* -** Return the bitmask for the given cursor number. Return 0 if -** iCursor is not in the set. +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +** +** If all the children of a term are disabled, then that term is also +** automatically disabled. In this way, terms get disabled if derived +** virtual terms are tested first. For example: +** +** x GLOB 'abc*' AND x>='abc' AND x<'acd' +** \___________/ \______/ \_____/ +** parent child1 child2 +** +** Only the parent term was in the original WHERE clause. The child1 +** and child2 terms were added by the LIKE optimization. If both of +** the virtual child terms are valid, then testing of the parent can be +** skipped. +** +** Usually the parent term is marked as TERM_CODED. But if the parent +** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. +** The TERM_LIKECOND marking indicates that the term should be coded inside +** a conditional such that is only evaluated on the second pass of a +** LIKE-optimization loop, when scanning BLOBs instead of strings. */ -static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ - int i; - assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ - if( pMaskSet->ix[i]==iCursor ){ - return MASKBIT(i); +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + int nLoop = 0; + while( pTerm + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ + pTerm->wtFlags |= TERM_LIKECOND; + }else{ + pTerm->wtFlags |= TERM_CODED; } + if( pTerm->iParent<0 ) break; + pTerm = &pTerm->pWC->a[pTerm->iParent]; + pTerm->nChild--; + if( pTerm->nChild!=0 ) break; + nLoop++; } - return 0; } /* -** Create a new mask for cursor iCursor. +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. ** -** There is one cursor per table in the FROM clause. The number of -** tables in the FROM clause is limited by a test early in the -** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] -** array will never overflow. -*/ -static void createMask(WhereMaskSet *pMaskSet, int iCursor){ - assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); - pMaskSet->ix[pMaskSet->n++] = iCursor; -} - -/* -** These routines walk (recursively) an expression tree and generate -** a bitmask indicating which tables are used in that expression -** tree. +** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_BLOB, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. */ -static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); -static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); -static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask = 0; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = getMask(pMaskSet, p->iTable); - return mask; +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; } - mask = exprTableUsage(pMaskSet, p->pRight); - mask |= exprTableUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ - mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); - }else{ - mask |= exprListTableUsage(pMaskSet, p->x.pList); + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + n--; + base++; + zAff++; } - return mask; -} -static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ - int i; - Bitmask mask = 0; - if( pList ){ - for(i=0; inExpr; i++){ - mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); - } + while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + n--; } - return mask; -} -static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ - Bitmask mask = 0; - while( pS ){ - SrcList *pSrc = pS->pSrc; - mask |= exprListTableUsage(pMaskSet, pS->pEList); - mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); - mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); - mask |= exprTableUsage(pMaskSet, pS->pWhere); - mask |= exprTableUsage(pMaskSet, pS->pHaving); - if( ALWAYS(pSrc!=0) ){ - int i; - for(i=0; inSrc; i++){ - mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); - } - } - pS = pS->pPrior; + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); } - return mask; } -/* -** Return TRUE if the given operator is one of the operators that is -** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" -*/ -static int allowedOp(int op){ - assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; -} /* -** Commute a comparison operator. Expressions of the form "X op Y" -** are converted into "Y op X". +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. ** -** If left/right precedence rules come into play when determining the -** collating sequence, then COLLATE operators are adjusted to ensure -** that the collating sequence does not change. For example: -** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_Collate flag -** is not commuted. +** The current value for the constraint is left in register iReg. +** +** For a constraint of the form X=expr, the expression is evaluated and its +** result is left on the stack. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_Collate); - u16 expLeft = (pExpr->pLeft->flags & EP_Collate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - if( expRight==expLeft ){ - /* Either X and Y both have COLLATE operator or neither do */ - if( expRight ){ - /* Both X and Y have COLLATE operators. Make sure X is always - ** used by clearing the EP_Collate flag from Y. */ - pExpr->pRight->flags &= ~EP_Collate; - }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ - /* Neither X nor Y have COLLATE operators, but X has a non-default - ** collating sequence. So add the EP_Collate marker on X to cause - ** it to be searched first. */ - pExpr->pLeft->flags |= EP_Collate; +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( iTarget>0 ); + if( pX->op==TK_EQ || pX->op==TK_IS ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; } + assert( pX->op==TK_IN ); + iReg = iTarget; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + pLevel->u.in.nIn++; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + pIn += pLevel->u.in.nIn - 1; + pIn->iCur = iTab; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); + } + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); + }else{ + pLevel->u.in.nIn = 0; + } +#endif } - SWAP(Expr*,pExpr->pRight,pExpr->pLeft); - if( pExpr->op>=TK_GT ){ - assert( TK_LT==TK_GT+2 ); - assert( TK_GE==TK_LE+2 ); - assert( TK_GT>TK_EQ ); - assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); - pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; - } + disableTerm(pLevel, pTerm); + return iReg; } /* -** Translate from TK_xx operator to WO_xx bitmask. +** Generate code that will evaluate all == and IN constraints for an +** index scan. +** +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. +** +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. +** +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use BLOB or NONE affinity are set to +** SQLITE_AFF_BLOB. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has BLOB/NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_BLOB. */ -static u16 operatorMask(int op){ - u16 c; - assert( allowedOp(op) ); - if( op==TK_IN ){ - c = WO_IN; - }else if( op==TK_ISNULL ){ - c = WO_ISNULL; - }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ + + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); + if( !zAff ){ + pParse->db->mallocFailed = 1; } - assert( op!=TK_ISNULL || c==WO_ISNULL ); - assert( op!=TK_IN || c==WO_IN ); - assert( op!=TK_EQ || c==WO_EQ ); - assert( op!=TK_LT || c==WO_LT ); - assert( op!=TK_LE || c==WO_LE ); - assert( op!=TK_GT || c==WO_GT ); - assert( op!=TK_GE || c==WO_GE ); - return c; -} -/* -** Advance to the next WhereTerm that matches according to the criteria -** established when the pScan object was initialized by whereScanInit(). -** Return NULL if there are no more matching WhereTerms. -*/ -static WhereTerm *whereScanNext(WhereScan *pScan){ - int iCur; /* The cursor on the LHS of the term */ - int iColumn; /* The column on the LHS of the term. -1 for IPK */ - Expr *pX; /* An expression being tested */ - WhereClause *pWC; /* Shorthand for pScan->pWC */ - WhereTerm *pTerm; /* The term being tested */ - int k = pScan->k; /* Where to start scanning */ + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]==XN_EXPR ); + VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); + } + } - while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aEquiv[pScan->iEquiv-2]; - iColumn = pScan->aEquiv[pScan->iEquiv-1]; - while( (pWC = pScan->pWC)!=0 ){ - for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ - if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn - && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - ){ - if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquivaEquiv) - ){ - int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); - for(j=0; jnEquiv; j+=2){ - if( pScan->aEquiv[j]==pX->iTable - && pScan->aEquiv[j+1]==pX->iColumn ){ - break; - } - } - if( j==pScan->nEquiv ){ - pScan->aEquiv[j] = pX->iTable; - pScan->aEquiv[j+1] = pX->iColumn; - pScan->nEquiv += 2; - } - } - if( (pTerm->eOperator & pScan->opMask)!=0 ){ - /* Verify the affinity and collating sequence match */ - if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; - Parse *pParse = pWC->pWInfo->pParse; - pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; - } - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ - continue; - } - } - if( (pTerm->eOperator & WO_EQ)!=0 - && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aEquiv[0] - && pX->iColumn==pScan->aEquiv[1] - ){ - continue; - } - pScan->k = k+1; - return pTerm; - } + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IN ); + if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ + zAff[j] = SQLITE_AFF_BLOB; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_BLOB; } } - pScan->pWC = pScan->pWC->pOuter; - k = 0; } - pScan->pWC = pScan->pOrigWC; - k = 0; - pScan->iEquiv += 2; } - return 0; + *pzAff = zAff; + return regBase; } /* -** Initialize a WHERE clause scanner object. Return a pointer to the -** first match. Return NULL if there are no matches. -** -** The scanner will be searching the WHERE clause pWC. It will look -** for terms of the form "X " where X is column iColumn of table -** iCur. The must be one of the operators described by opMask. -** -** If the search is for X and the WHERE clause contains terms of the -** form X=Y then this routine might also return terms of the form -** "Y ". The number of levels of transitivity is limited, -** but is enough to handle most commonly occurring SQL statements. +** If the most recently coded instruction is a constant range contraint +** that originated from the LIKE optimization, then change the P3 to be +** pLoop->iLikeRepCntr and set P5. ** -** If X is not the INTEGER PRIMARY KEY then X must be compatible with -** index pIdx. +** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range +** expression: "x>='ABC' AND x<'abd'". But this requires that the range +** scan loop run twice, once for strings and a second time for BLOBs. +** The OP_String opcodes on the second pass convert the upper and lower +** bound string contants to blobs. This routine makes the necessary changes +** to the OP_String opcodes for that to happen. */ -static WhereTerm *whereScanInit( - WhereScan *pScan, /* The WhereScan object being initialized */ - WhereClause *pWC, /* The WHERE clause to be scanned */ - int iCur, /* Cursor to scan for */ - int iColumn, /* Column to scan for */ - u32 opMask, /* Operator(s) to scan for */ - Index *pIdx /* Must be compatible with this index */ +static void whereLikeOptimizationStringFixup( + Vdbe *v, /* prepared statement under construction */ + WhereLevel *pLevel, /* The loop that contains the LIKE operator */ + WhereTerm *pTerm /* The upper or lower bound just coded */ ){ - int j; - - /* memset(pScan, 0, sizeof(*pScan)); */ - pScan->pOrigWC = pWC; - pScan->pWC = pWC; - if( pIdx && iColumn>=0 ){ - pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>pIdx->nColumn) ) return 0; - } - pScan->zCollName = pIdx->azColl[j]; - }else{ - pScan->idxaff = 0; - pScan->zCollName = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + VdbeOp *pOp; + assert( pLevel->iLikeRepCntr>0 ); + pOp = sqlite3VdbeGetOp(v, -1); + assert( pOp!=0 ); + assert( pOp->opcode==OP_String8 + || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); + pOp->p3 = pLevel->iLikeRepCntr; + pOp->p5 = 1; } - pScan->opMask = opMask; - pScan->k = 0; - pScan->aEquiv[0] = iCur; - pScan->aEquiv[1] = iColumn; - pScan->nEquiv = 2; - pScan->iEquiv = 2; - return whereScanNext(pScan); } + /* -** Search for a term in the WHERE clause that is of the form "X " -** where X is a reference to the iColumn of table iCur and is one of -** the WO_xx operator codes specified by the op parameter. -** Return a pointer to the term. Return 0 if not found. -** -** The term returned might by Y= if there is another constraint in -** the WHERE clause that specifies that X=Y. Any such constraints will be -** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aEquiv[] array holds X and all its equivalents, with each SQL variable -** taking up two slots in aEquiv[]. The first slot is for the cursor number -** and the second is for the column number. There are 22 slots in aEquiv[] -** so that means we can look for X plus up to 10 other equivalent values. -** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 -** and ... and A9=A10 and A10=. -** -** If there are multiple terms in the WHERE clause of the form "X " -** then try for the one with no dependencies on - in other words where -** is a constant expression of some kind. Only return entries of -** the form "X Y" where Y is a column in another table if no terms of -** the form "X " exist. If no terms with a constant RHS -** exist, try to return a term that does not use WO_EQUIV. +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. */ -static WhereTerm *findTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ ){ - WhereTerm *pResult = 0; - WhereTerm *p; - WhereScan scan; + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ - p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); - while( p ){ - if( (p->prereqRight & notReady)==0 ){ - if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ - return p; - } - if( pResult==0 ) pResult = p; - } - p = whereScanNext(&scan); - } - return pResult; -} + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = &pWInfo->sWC; + db = pParse->db; + pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); -/* Forward reference */ -static void exprAnalyze(SrcList*, WhereClause*, int); + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); -/* -** Call exprAnalyze on all terms in a WHERE clause. -*/ -static void exprAnalyzeAll( - SrcList *pTabList, /* the FROM clause */ - WhereClause *pWC /* the WHERE clause to be analyzed */ -){ - int i; - for(i=pWC->nTerm-1; i>=0; i--){ - exprAnalyze(pTabList, pWC, i); + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); } -} -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION -/* -** Check to see if the given expression is a LIKE or GLOB operator that -** can be optimized using inequality constraints. Return TRUE if it is -** so and false if not. -** -** In order for the operator to be optimizible, the RHS must be a string -** literal that does not begin with a wildcard. -*/ -static int isLikeOrGlob( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* Test this expression */ - Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ - int *pisComplete, /* True if the only wildcard is % in the last character */ - int *pnoCase /* True if uppercase is equivalent to lowercase */ -){ - const char *z = 0; /* String on RHS of LIKE operator */ - Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ - ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ - int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ - sqlite3 *db = pParse->db; /* Database connection */ - sqlite3_value *pVal = 0; - int op; /* Opcode of pRight */ + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ - return 0; - } -#ifdef SQLITE_EBCDIC - if( *pnoCase ) return 0; -#endif - pList = pExpr->x.pList; - pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; - pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); - op = pRight->op; - if( op==TK_VARIABLE ){ - Vdbe *pReprepare = pParse->pReprepare; - int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); - if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); - } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); - assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); - }else if( op==TK_STRING ){ - z = pRight->u.zToken; - } - if( z ){ - cnt = 0; - while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ - cnt++; + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } } - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ - Expr *pPrefix; - *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; - *ppPrefix = pPrefix; - if( op==TK_VARIABLE ){ - Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); - if( *pisComplete && pRight->u.zToken[1] ){ - /* If the rhs of the LIKE expression is a variable, and the current - ** value of the variable means there is no need to invoke the LIKE - ** function, then no OP_Variable will be added to the program. - ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To work around them, add a dummy OP_Variable here. - */ - int r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCodeTarget(pParse, pRight, r1); - sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); - sqlite3ReleaseTempReg(pParse, r1); - } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); } - }else{ - z = 0; } - } - - sqlite3ValueFree(pVal); - return (z!=0); -} -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ - - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Check to see if the given expression is of the form -** -** column MATCH expr -** -** If it is then return TRUE. If not, return FALSE. -*/ -static int isMatchOfColumn( - Expr *pExpr /* Test this expression */ -){ - ExprList *pList; - - if( pExpr->op!=TK_FUNCTION ){ - return 0; - } - if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ - return 0; - } - pList = pExpr->x.pList; - if( pList->nExpr!=2 ){ - return 0; - } - if( pList->a[1].pExpr->op != TK_COLUMN ){ - return 0; - } - return 1; -} + pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse); + }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** If the pBase expression originated in the ON or USING clause of -** a join, then transfer the appropriate markings over to derived. -*/ -static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - if( pDerived ){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; - } -} + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) -/* -** Analyze a term that consists of two or more OR-connected -** subterms. So in: -** -** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) -** ^^^^^^^^^^^^^^^^^^^^ -** -** This routine analyzes terms such as the middle term in the above example. -** A WhereOrTerm object is computed and attached to the term under -** analysis, regardless of the outcome of the analysis. Hence: -** -** WhereTerm.wtFlags |= TERM_ORINFO -** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object -** -** The term being analyzed must have two or more of OR-connected subterms. -** A single subterm might be a set of AND-connected sub-subterms. -** Examples of terms under analysis: -** -** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 -** (B) x=expr1 OR expr2=x OR x=expr3 -** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) -** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') -** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) -** -** CASE 1: -** -** If all subterms are of the form T.C=expr for some single column of C and -** a single table T (as shown in example B above) then create a new virtual -** term that is an equivalent IN expression. In other words, if the term -** being analyzed is: -** -** x = expr1 OR expr2 = x OR x = expr3 -** -** then create a new virtual term like this: -** -** x IN (expr1,expr2,expr3) -** -** CASE 2: -** -** If all subterms are indexable by a single table T, then set -** -** WhereTerm.eOperator = WO_OR -** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T -** -** A subterm is "indexable" if it is of the form -** "T.C " where C is any column of table T and -** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". -** A subterm is also indexable if it is an AND of two or more -** subsubterms at least one of which is indexable. Indexable AND -** subterms have their eOperator set to WO_AND and they have -** u.pAndInfo set to a dynamically allocated WhereAndTerm object. -** -** From another point of view, "indexable" means that the subterm could -** potentially be used with an index if an appropriate index exists. -** This analysis does not consider whether or not the index exists; that -** is decided elsewhere. This analysis only looks at whether subterms -** appropriate for indexing exist. -** -** All examples A through E above satisfy case 2. But if a term -** also satisfies case 1 (such as B) we know that the optimizer will -** always prefer case 1, so in that case we pretend that case 2 is not -** satisfied. -** -** It might be the case that multiple tables are indexable. For example, -** (E) above is indexable on tables P, Q, and R. -** -** Terms that satisfy case 2 are candidates for lookup by using -** separate indices to find rowids for each subterm and composing -** the union of all rowids using a RowSet object. This is similar -** to "bitmap indices" in other database engines. -** -** OTHERWISE: -** -** If neither case 1 nor case 2 apply, then leave the eOperator set to -** zero. This term is not useful for search. -*/ -static void exprAnalyzeOrTerm( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the complete WHERE clause */ - int idxTerm /* Index of the OR-term to be analyzed */ -){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - Parse *pParse = pWInfo->pParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ - Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - int i; /* Loop counters */ - WhereClause *pOrWc; /* Breakup of pTerm into subterms */ - WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ - WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ - Bitmask chngToIN; /* Tables that might satisfy case 1 */ - Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ - /* - ** Break the OR clause into its separate subterms. The subterms are - ** stored in a WhereClause structure containing within the WhereOrInfo - ** object that is attached to the original OR clause term. - */ - assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); - assert( pExpr->op==TK_OR ); - pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); - if( pOrInfo==0 ) return; - pTerm->wtFlags |= TERM_ORINFO; - pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWInfo); - whereSplit(pOrWc, pExpr, TK_OR); - exprAnalyzeAll(pSrc, pOrWc); - if( db->mallocFailed ) return; - assert( pOrWc->nTerm>=2 ); + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - /* - ** Compute the set of tables that might satisfy cases 1 or 2. - */ - indexable = ~(Bitmask)0; - chngToIN = ~(Bitmask)0; - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ - if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ - WhereAndInfo *pAndInfo; - assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); - chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); - if( pAndInfo ){ - WhereClause *pAndWC; - WhereTerm *pAndTerm; - int j; - Bitmask b = 0; - pOrTerm->u.pAndInfo = pAndInfo; - pOrTerm->wtFlags |= TERM_ANDINFO; - pOrTerm->eOperator = WO_AND; - pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pWInfo); - whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); - exprAnalyzeAll(pSrc, pAndWC); - pAndWC->pOuter = pWC; - testcase( db->mallocFailed ); - if( !db->mallocFailed ){ - for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ - assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); - } - } - } - indexable &= b; - } - }else if( pOrTerm->wtFlags & TERM_COPIED ){ - /* Skip this term for now. We revisit it when we process the - ** corresponding TERM_VIRTUAL term */ + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); }else{ - Bitmask b; - b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); - if( pOrTerm->wtFlags & TERM_VIRTUAL ){ - WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); - } - indexable &= b; - if( (pOrTerm->eOperator & WO_EQ)==0 ){ - chngToIN = 0; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + sqlite3ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; }else{ - chngToIN &= b; + testOp = bRev ? OP_Lt : OP_Gt; } + disableTerm(pLevel, pEnd); } - } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ - /* - ** Record the set of tables that satisfy case 2. The set might be - ** empty. - */ - pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->nSkip ); - /* - ** chngToIN holds a set of tables that *might* satisfy case 1. But - ** we have to do some additional checking to see if case 1 really - ** is satisfied. - ** - ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means - ** that there is no possibility of transforming the OR clause into an - ** IN operator because one or more terms in the OR clause contain - ** something other than == on a column in the single table. The 1-bit - ** case means that every term of the OR clause is of the form - ** "table.column=expr" for some single table. The one bit that is set - ** will correspond to the common table. We still need to check to make - ** sure the same column is used on all terms. The 2-bit case is when - ** the all terms are of the form "table1.column=table2.column". It - ** might be possible to form an IN operator with either table1.column - ** or table2.column as the LHS if either is common to every term of - ** the OR clause. - ** - ** Note that terms of the form "table.column1=table.column2" (the - ** same table on both sizes of the ==) cannot be optimized. - */ - if( chngToIN ){ - int okToChngToIN = 0; /* True if the conversion to IN is valid */ - int iColumn = -1; /* Column index on lhs of IN operator */ - int iCursor = -1; /* Table cursor common to all terms */ - int j = 0; /* Loop counter */ + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) + ){ + assert( pLoop->nSkip==0 ); + bSeekPastNull = 1; + nExtraReg = 1; + } - /* Search for a table and column that appears on one side or the - ** other of the == operator in every subterm. That table and column - ** will be recorded in iCursor and iColumn. There might not be any - ** such table and column. Set okToChngToIN if an appropriate table - ** and column is found but leave okToChngToIN false if not found. + /* Find any inequality constraint terms for the start and end + ** of the range. */ - for(j=0; j<2 && !okToChngToIN; j++){ - pOrTerm = pOrWc->a; - for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; - if( pOrTerm->leftCursor==iCursor ){ - /* This is the 2-bit case and we are on the second iteration and - ** current term is from the first iteration. So skip this term. */ - assert( j==1 ); - continue; - } - if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ - /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term - ** and use its inversion. */ - testcase( pOrTerm->wtFlags & TERM_COPIED ); - testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); - assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); - continue; - } - iColumn = pOrTerm->u.leftColumn; - iCursor = pOrTerm->leftCursor; - break; - } - if( i<0 ){ - /* No candidate table+column was found. This can only occur - ** on the second iteration */ - assert( j==1 ); - assert( IsPowerOfTwo(chngToIN) ); - assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); - break; + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = 1; + /* Like optimization range constraints always occur in pairs */ + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = 1; + if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ + assert( pRangeStart!=0 ); /* LIKE opt constraints */ + assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ + pLevel->iLikeRepCntr = ++pParse->nMem; + testcase( bRev ); + testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); + sqlite3VdbeAddOp2(v, OP_Integer, + bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC), + pLevel->iLikeRepCntr); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); } - testcase( j==1 ); - - /* We have found a candidate table and column. Check to see if that - ** table and column is common to every term in the OR clause */ - okToChngToIN = 1; - for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ - okToChngToIN = 0; - }else{ - int affLeft, affRight; - /* If the right-hand side is also a column, then the affinities - ** of both right and left sides must be such that no type - ** conversions are required on the right. (Ticket #2249) - */ - affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); - affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); - if( affRight!=0 && affRight!=affLeft ){ - okToChngToIN = 0; - }else{ - pOrTerm->wtFlags |= TERM_OR_OK; - } - } + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ + bSeekPastNull = 1; } } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - /* At this point, okToChngToIN is true if original pTerm satisfies - ** case 1. In that case, construct a new virtual term that is - ** pTerm converted into an IN operator. + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. */ - if( okToChngToIN ){ - Expr *pDup; /* A transient duplicate expression */ - ExprList *pList = 0; /* The RHS of the IN operator */ - Expr *pLeft = 0; /* The LHS of the IN operator */ - Expr *pNew; /* The complete IN operator */ + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator & WO_EQ ); - assert( pOrTerm->leftCursor==iCursor ); - assert( pOrTerm->u.leftColumn==iColumn ); - pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); - pLeft = pOrTerm->pExpr->pLeft; - } - assert( pLeft!=0 ); - pDup = sqlite3ExprDup(db, pLeft, 0); - pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); - if( pNew ){ - int idxNew; - transferJoinMarkings(pNew, pExpr); - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - pNew->x.pList = pList; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; - }else{ - sqlite3ExprListDelete(db, pList); - } - pTerm->eOperator = WO_NOOP; /* case 1 trumps case 2 */ + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) + ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); } - } -} -#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ -/* -** The input to this routine is an WhereTerm structure with only the -** "pExpr" field filled in. The job of this routine is to analyze the -** subexpression and populate all the other fields of the WhereTerm -** structure. -** -** If the expression is of the form " X" it gets commuted -** to the standard form of "X ". -** -** If the expression is of the form "X Y" where both X and Y are -** columns, then the original expression is unchanged and a new virtual -** term of the form "Y X" is added to the WHERE clause and -** analyzed separately. The original term is marked with TERM_COPIED -** and the new term is marked with TERM_DYNAMIC (because it's pExpr -** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it -** is a commuted copy of a prior term.) The original term has nChild=1 -** and the copy has idxParent set to the index of the original term. -*/ -static void exprAnalyze( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the WHERE clause */ - int idxTerm /* Index of the term to be analyzed */ -){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - WhereTerm *pTerm; /* The term to be analyzed */ - WhereMaskSet *pMaskSet; /* Set of table index masks */ - Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ - Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ - Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ - int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ - int noCase = 0; /* LIKE/GLOB distinguishes case */ - int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWInfo->pParse; /* Parsing context */ - sqlite3 *db = pParse->db; /* Database connection */ + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; - if( db->mallocFailed ){ - return; - } - pTerm = &pWC->a[idxTerm]; - pMaskSet = &pWInfo->sMaskSet; - pExpr = pTerm->pExpr; - assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); - prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); - op = pExpr->op; - if( op==TK_IN ){ - assert( pExpr->pRight==0 ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); - }else{ - pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); - } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; - }else{ - pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); - } - prereqAll = exprTableUsage(pMaskSet, pExpr); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ - } - pTerm->prereqAll = prereqAll; - pTerm->leftCursor = -1; - pTerm->iParent = -1; - pTerm->eOperator = 0; - if( allowedOp(op) ){ - Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); - Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); - u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op) & opMask; - } - if( pRight && pRight->op==TK_COLUMN ){ - WhereTerm *pNew; - Expr *pDup; - u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ - if( pTerm->leftCursor>=0 ){ - int idxNew; - pDup = sqlite3ExprDup(db, pExpr, 0); - if( db->mallocFailed ){ - sqlite3ExprDelete(db, pDup); - return; + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zStartAff ){ + if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_BLOB){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_BLOB. */ + zStartAff[nEq] = SQLITE_AFF_BLOB; } - idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); - if( idxNew==0 ) return; - pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; - pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; - pTerm->wtFlags |= TERM_COPIED; - if( pExpr->op==TK_EQ - && !ExprHasProperty(pExpr, EP_FromJoin) - && OptimizationEnabled(db, SQLITE_Transitive) - ){ - pTerm->eOperator |= WO_EQUIV; - eExtraOp = WO_EQUIV; + if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ + zStartAff[nEq] = SQLITE_AFF_BLOB; } - }else{ - pDup = pExpr; - pNew = pTerm; - } - exprCommute(pParse, pDup); - pLeft = sqlite3ExprSkipCollate(pDup->pLeft); - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; - testcase( (prereqLeft | extraRight) != prereqLeft ); - pNew->prereqRight = prereqLeft | extraRight; - pNew->prereqAll = prereqAll; - pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + } + nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + }else if( bSeekPastNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; } - } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); -#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION - /* If a term is the BETWEEN operator, create two new virtual terms - ** that define the range that the BETWEEN implements. For example: - ** - ** a BETWEEN b AND c - ** - ** is converted into: - ** - ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) - ** - ** The two new terms are added onto the end of the WhereClause object. - ** The new terms are "dynamic" and are children of the original BETWEEN - ** term. That means that if the BETWEEN term is coded, the children are - ** skipped. Or, if the children are satisfied by an index, the original - ** BETWEEN term is skipped. - */ - else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ - ExprList *pList = pExpr->x.pList; - int i; - static const u8 ops[] = {TK_GE, TK_LE}; - assert( pList!=0 ); - assert( pList->nExpr==2 ); - for(i=0; i<2; i++){ - Expr *pNewExpr; - int idxNew; - pNewExpr = sqlite3PExpr(pParse, ops[i], - sqlite3ExprDup(db, pExpr->pLeft, 0), - sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); - transferJoinMarkings(pNewExpr, pExpr); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_BLOB + && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) + ){ + codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); + } + nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; } - pTerm->nChild = 2; - } -#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ + sqlite3DbFree(db, zStartAff); -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) - /* Analyze a term that is composed of two or more subterms connected by - ** an OR operator. - */ - else if( pExpr->op==TK_OR ){ - assert( pWC->op==TK_AND ); - exprAnalyzeOrTerm(pSrc, pWC, idxTerm); - pTerm = &pWC->a[idxTerm]; - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION - /* Add constraints to reduce the search space on a LIKE or GLOB - ** operator. - ** - ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints - ** - ** x>='abc' AND x<'abd' AND x LIKE 'abc%' - ** - ** The last character of the prefix "abc" is incremented to form the - ** termination condition "abd". - */ - if( pWC->op==TK_AND - && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) - ){ - Expr *pLeft; /* LHS of LIKE/GLOB operator */ - Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ - Expr *pNewExpr1; - Expr *pNewExpr2; - int idxNew1; - int idxNew2; - Token sCollSeqName; /* Name of collating sequence */ + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } - pLeft = pExpr->x.pList->a[1].pExpr; - pStr2 = sqlite3ExprDup(db, pStr1, 0); - if( !db->mallocFailed ){ - u8 c, *pC; /* Last character before the first wildcard */ - pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; - c = *pC; - if( noCase ){ - /* The point is to increment the last character before the first - ** wildcard. But if we increment '@', that will push it into the - ** alphabetic range where case conversions will mess up the - ** inequality. To avoid this, make sure to also run the full - ** LIKE on all candidate expressions by clearing the isComplete flag - */ - if( c=='A'-1 ) isComplete = 0; - c = sqlite3UpperToLower[c]; + /* Seek the table cursor, if required */ + disableTerm(pLevel, pRangeStart); + disableTerm(pLevel, pRangeEnd); + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + if( pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ } - *pC = c + 1; + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; - sCollSeqName.n = 6; - pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), - pStr1, 0); - transferJoinMarkings(pNewExpr1, pExpr); - idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), - pStr2, 0); - transferJoinMarkings(pNewExpr2, pExpr); - idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew2==0 ); - exprAnalyze(pSrc, pWC, idxNew2); - pTerm = &pWC->a[idxTerm]; - if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + + /* Record the instruction used to terminate the loop. Disable + ** WHERE clause terms made redundant by the index range scan. + */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; } - } -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + }else -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Add a WO_MATCH auxiliary term to the constraint set if the - ** current expression is of the form: column MATCH expr. - ** This information is used by the xBestIndex methods of - ** virtual tables. The native query optimizer does not attempt - ** to do anything with MATCH functions. - */ - if( isMatchOfColumn(pExpr) ){ - int idxNew; - Expr *pRight, *pLeft; - WhereTerm *pNewTerm; - Bitmask prereqColumn, prereqExpr; +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ - pRight = pExpr->x.pList->a[0].pExpr; - pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = exprTableUsage(pMaskSet, pRight); - prereqColumn = exprTableUsage(pMaskSet, pLeft); - if( (prereqExpr & prereqColumn)==0 ){ - Expr *pNewExpr; - pNewExpr = sqlite3PExpr(pParse, TK_MATCH, - 0, sqlite3ExprDup(db, pRight, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; - pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; + + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; } - } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the - ** form "x IS NOT NULL" can sometimes be evaluated more efficiently - ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a - ** virtual term of that form. - ** - ** Note that the virtual term must be tagged with TERM_VNULL. This - ** TERM_VNULL tag will suppress the not-null check at the beginning - ** of the loop. Without the TERM_VNULL flag, the not-null check at - ** the start of the loop will prevent any results from being returned. - */ - if( pExpr->op==TK_NOTNULL - && pExpr->pLeft->op==TK_COLUMN - && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pNewExpr; - Expr *pLeft = pExpr->pLeft; - int idxNew; - WhereTerm *pNewTerm; + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + regRowid = ++pParse->nMem; + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - pNewExpr = sqlite3PExpr(pParse, TK_GT, - sqlite3ExprDup(db, pLeft, 0), - sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + } + } - idxNew = whereClauseInsert(pWC, pNewExpr, - TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); - if( idxNew ){ - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = 0; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; - pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY + | WHERE_NO_AUTOINDEX; + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int jmp1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } + /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + int addrExplain = sqlite3WhereExplainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int r; + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + int rx; + rx = sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur,r+iPk,0); + if( rx!=r+iPk ){ + sqlite3VdbeAddOp2(v, OP_SCopy, rx, r+iPk); + } + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } + + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; + }else{ + pCov = 0; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeGoto(v, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 6: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + if( pTabItem->fg.isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - /* Prevent ON clause terms of a LEFT JOIN from being used to drive - ** an index for tables to the left of the join. +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. */ - pTerm->prereqRight |= extraRight; -} + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + assert( pLevel->iLikeRepCntr>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr); + VdbeCoverage(v); + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; + } -/* -** This function searches pList for an entry that matches the iCol-th column -** of index pIdx. -** -** If such an expression is found, its index in pList->a[] is returned. If -** no expression is found, -1 is returned. -*/ -static int findIndexCol( - Parse *pParse, /* Parse context */ - ExprList *pList, /* Expression list to search */ - int iBase, /* Cursor for table associated with pIdx */ - Index *pIdx, /* Index to match column of */ - int iCol /* Column of index to match */ -){ - int i; - const char *zColl = pIdx->azColl[iCol]; + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, *pEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; + if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, + WO_EQ|WO_IN|WO_IS, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IS ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); + if( pEAlt ){ + *pEAlt = *pAlt->pExpr; + pEAlt->pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); + sqlite3StackFree(db, pEAlt); + } + } - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); - if( p->op==TK_COLUMN - && p->iColumn==pIdx->aiColumn[iCol] - && p->iTable==iBase - ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); - if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ - return i; + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; } } - return -1; + return pLevel->notReady; } +/************** End of wherecode.c *******************************************/ +/************** Begin file whereexpr.c ***************************************/ /* -** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. ** -** A DISTINCT list is redundant if the database contains some subset of -** columns that are unique and non-null. +** This file was originally part of where.c but was split out to improve +** readability and editabiliity. This file contains utility routines for +** analyzing Expr objects in the WHERE clause. */ -static int isDistinctRedundant( - Parse *pParse, /* Parsing context */ - SrcList *pTabList, /* The FROM clause */ - WhereClause *pWC, /* The WHERE clause */ - ExprList *pDistinct /* The result set that needs to be DISTINCT */ -){ - Table *pTab; - Index *pIdx; - int i; - int iBase; +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ - /* If there is more than one table or sub-select in the FROM clause of - ** this query, then it will not be possible to show that the DISTINCT - ** clause is redundant. */ - if( pTabList->nSrc!=1 ) return 0; - iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; +/* Forward declarations */ +static void exprAnalyze(SrcList*, WhereClause*, int); - /* If any of the expressions is an IPK column on table iBase, then return - ** true. Note: The (p->iTable==iBase) part of this test may be false if the - ** current SELECT is a correlated sub-query. - */ - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); - if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; - } +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} - /* Loop through all indices on the table, checking each to see if it makes - ** the DISTINCT qualifier redundant. It does so if: - ** - ** 1. The index is itself UNIQUE, and - ** - ** 2. All of the columns in the index are either part of the pDistinct - ** list, or else the WHERE clause contains a term of the form "col=X", - ** where X is a constant value. The collation sequences of the - ** comparison and select-list expressions must match those of the index. - ** - ** 3. All of those index columns for which the WHERE clause does not - ** contain a "col=X" term are subject to a NOT NULL constraint. - */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !IsUniqueIndex(pIdx) ) continue; - for(i=0; inKeyCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); - if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ - break; - } +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. +** +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. +*/ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pWInfo->pParse->db; + pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); } + pWC->a = pOld; + return 0; } - if( i==pIdx->nKeyCol ){ - /* This index implies that the DISTINCT qualifier is redundant. */ - return 1; + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } - - return 0; + pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; + }else{ + pTerm->truthProb = 1; + } + pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + return idx; } - /* -** Estimate the logarithm of the input value to base 2. +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" */ -static LogEst estLog(LogEst N){ - return N<=10 ? 0 : sqlite3LogEst(N) - 33; +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; } /* -** Two routines for printing the content of an sqlite3_index_info -** structure. Used for testing and debugging only. If neither -** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines -** are no-ops. +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +** +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on +** the left hand side of a comparison overrides any collation sequence +** attached to the right. For the same reason the EP_Collate flag +** is not commuted. */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", - i, - p->aConstraint[i].iColumn, - p->aConstraint[i].iTermOffset, - p->aConstraint[i].op, - p->aConstraint[i].usable); - } - for(i=0; inOrderBy; i++){ - sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); +static void exprCommute(Parse *pParse, Expr *pExpr){ + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); + assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } } -} -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", - i, - p->aConstraintUsage[i].argvIndex, - p->aConstraintUsage[i].omit); + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; } - sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); - sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); - sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); - sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); - sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } -#else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) -#endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* -** Return TRUE if the WHERE clause term pTerm is of a form where it -** could be used with an index to access pSrc, assuming an appropriate -** index existed. +** Translate from TK_xx operator to WO_xx bitmask. */ -static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - struct SrcList_item *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ -){ - char aff; - if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( (pTerm->eOperator & WO_EQ)==0 ) return 0; - if( (pTerm->prereqRight & notReady)!=0 ) return 0; - if( pTerm->u.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; - if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; - return 1; +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else if( op==TK_IS ){ + c = WO_IS; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + assert( op!=TK_IS || c==WO_IS ); + return c; } -#endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* -** Generate code to construct the Index object for an automatic index -** and to set up the WhereLevel object pLevel so that the code generator -** makes use of the automatic index. +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. The LHS must be a column +** that may only be NULL, a string, or a BLOB, never a number. (This means +** that virtual tables cannot participate in the LIKE optimization.) The +** collating sequence for the column on the LHS must be appropriate for +** the operator. */ -static void constructAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereLevel *pLevel /* Write new index here */ +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - int nKeyCol; /* Number of columns in the constructed index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Index *pIdx; /* Object describing the transient index */ - Vdbe *v; /* Prepared statement under construction */ - int addrInit; /* Address of the initialization bypass jump */ - Table *pTable; /* The table being indexed */ - int addrTop; /* Top of the index fill loop */ - int regRecord; /* Register holding an index record */ - int n; /* Column counter */ - int i; /* Loop counter */ - int mxBitCol; /* Maximum column in pSrc->colUsed */ - CollSeq *pColl; /* Collating sequence to on a column */ - WhereLoop *pLoop; /* The Loop object */ - char *zNotUsed; /* Extra space on the end of pIdx */ - Bitmask idxCols; /* Bitmap of columns used for indexing */ - Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ - - /* Generate code to skip over the creation and initialization of the - ** transient index on 2nd and subsequent iterations of the loop. */ - v = pParse->pVdbe; - assert( v!=0 ); - addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); - - /* Count the number of columns that will be added to the index - ** and used to match WHERE clause constraints */ - nKeyCol = 0; - pTable = pSrc->pTab; - pWCEnd = &pWC->a[pWC->nTerm]; - pLoop = pLevel->pWLoop; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - if( !sentWarning ){ - sqlite3_log(SQLITE_WARNING_AUTOINDEX, - "automatic index on %s(%s)", pTable->zName, - pTable->aCol[iCol].zName); - sentWarning = 1; - } - if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; - pLoop->aLTerm[nKeyCol++] = pTerm; - idxCols |= cMask; - } - } - } - assert( nKeyCol>0 ); - pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; - pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED - | WHERE_AUTO_INDEX; + const char *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + int c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + char wc[3]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ - /* Count the number of additional columns needed to create a - ** covering index. A "covering index" is an index that contains all - ** columns that are needed by the query. With a covering index, the - ** original table never needs to be accessed. Automatic indices must - ** be a covering index because the index will not be updated if the - ** original table changes and the index and table cannot both be used - ** if they go out of sync. - */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; - testcase( pTable->nCol==BMS-1 ); - testcase( pTable->nCol==BMS-2 ); - for(i=0; icolUsed & MASKBIT(BMS-1) ){ - nKeyCol += pTable->nCol - BMS + 1; +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ + ){ + /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must + ** be the name of an indexed column with TEXT affinity. */ + return 0; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; + assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - /* Construct the Index object to describe this index */ - pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; - pLoop->u.btree.pIndex = pIdx; - pIdx->zName = "auto-index"; - pIdx->pTable = pTable; - n = 0; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS-1 ); - testcase( iCol==BMS ); - if( (idxCols & cMask)==0 ){ - Expr *pX = pTerm->pExpr; - idxCols |= cMask; - pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY"; - n++; - } + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); + op = pRight->op; + if( op==TK_VARIABLE ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = (char *)sqlite3_value_text(pVal); } + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = pRight->u.zToken; } - assert( (u32)n==pLoop->u.btree.nEq ); - - /* Add additional columns needed to make the automatic index into - ** a covering index */ - for(i=0; iaiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; + if( z ){ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; } - } - if( pSrc->colUsed & MASKBIT(BMS-1) ){ - for(i=BMS-1; inCol; i++){ - pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + Expr *pPrefix; + *pisComplete = c==wc[0] && z[cnt+1]==0; + pPrefix = sqlite3Expr(db, TK_STRING, z); + if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + *ppPrefix = pPrefix; + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To work around them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); + } + } + }else{ + z = 0; } } - assert( n==nKeyCol ); - pIdx->aiColumn[n] = -1; - pIdx->azColl[n] = "BINARY"; - /* Create the automatic index */ - assert( pLevel->iIdxCur>=0 ); - pLevel->iIdxCur = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "for %s", pTable->zName)); - - /* Fill the automatic index with content */ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); - regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); - sqlite3VdbeJumpHere(v, addrTop); - sqlite3ReleaseTempReg(pParse, regRecord); - - /* Jump here when skipping the initialization */ - sqlite3VdbeJumpHere(v, addrInit); + sqlite3ValueFree(pVal); + return (z!=0); } -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Allocate and populate an sqlite3_index_info structure. It is the -** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). +** Check to see if the given expression is of the form +** +** column MATCH expr +** +** If it is then return TRUE. If not, return FALSE. */ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy +static int isMatchOfColumn( + Expr *pExpr /* Test this expression */ ){ - int i, j; - int nTerm; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_orderby *pIdxOrderBy; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int nOrderBy; - sqlite3_index_info *pIdxInfo; + ExprList *pList; - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - nTerm++; + if( pExpr->op!=TK_FUNCTION ){ + return 0; + } + if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ + return 0; + } + pList = pExpr->x.pList; + if( pList->nExpr!=2 ){ + return 0; + } + if( pList->a[1].pExpr->op != TK_COLUMN ){ + return 0; } + return 1; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ - /* If the ORDER BY clause contains only columns in the current - ** virtual table then allocate space for the aOrderBy part of - ** the sqlite3_index_info structure. - */ - nOrderBy = 0; - if( pOrderBy ){ - int n = pOrderBy->nExpr; - for(i=0; ia[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; - } - if( i==n){ - nOrderBy = n; - } +/* +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. +*/ +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; } +} - /* Allocate the sqlite3_index_info structure - */ - pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) - + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy ); - if( pIdxInfo==0 ){ - sqlite3ErrorMsg(pParse, "out of memory"); - return 0; +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + +/* +** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not +** a conjunction, then return just pTerm when N==0. If N is exceeds +** the number of available subterms, return NULL. +*/ +static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ + if( pTerm->eOperator!=WO_AND ){ + return N==0 ? pTerm : 0; } - - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. - */ - pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; - pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; - pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; - - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - u8 op; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - pIdxCons[j].iColumn = pTerm->u.leftColumn; - pIdxCons[j].iTermOffset = i; - op = (u8)pTerm->eOperator & WO_ALL; - if( op==WO_IN ) op = WO_EQ; - pIdxCons[j].op = op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); - j++; - } - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; - } - - return pIdxInfo; -} + if( Nu.pAndInfo->wc.nTerm ){ + return &pTerm->u.pAndInfo->wc.a[N]; + } + return 0; +} /* -** The table object reference passed as the second argument to this function -** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite3_index_info object that -** comes in as the 3rd argument to this function. +** Subterms pOne and pTwo are contained within WHERE clause pWC. The +** two subterms are in disjunction - they are OR-ed together. ** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. +** If these two terms are both of the form: "A op B" with the same +** A and B values but different operators and if the operators are +** compatible (if one is = and the other is <, for example) then +** add a new virtual AND term to pWC that is the combination of the +** two. ** -** Whether or not an error is returned, it is the responsibility of the -** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates -** that this is required. -*/ -static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int i; - int rc; - - TRACE_IDX_INPUTS(p); - rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); - - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; - }else if( !pVtab->zErrMsg ){ - sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); - }else{ - sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); - } - } - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; - - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); - } - } - - return pParse->nErr; -} -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the location of a particular key among all keys in an -** index. Store the results in aStat as follows: +** Some examples: ** -** aStat[0] Est. number of rows less than pVal -** aStat[1] Est. number of rows equal to pVal +** x x<=y +** x=y OR x=y --> x=y +** x<=y OR x x<=y ** -** Return SQLITE_OK on success. +** The following is NOT generated: +** +** xy --> x!=y */ -static void whereKeyStats( - Parse *pParse, /* Database connection */ - Index *pIdx, /* Index to consider domain of */ - UnpackedRecord *pRec, /* Vector of values to consider */ - int roundUp, /* Round up if true. Round down if false */ - tRowcnt *aStat /* OUT: stats written here */ +static void whereCombineDisjuncts( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* The complete WHERE clause */ + WhereTerm *pOne, /* First disjunct */ + WhereTerm *pTwo /* Second disjunct */ ){ - IndexSample *aSample = pIdx->aSample; - int iCol; /* Index of required stats in anEq[] etc. */ - int iMin = 0; /* Smallest sample not yet tested */ - int i = pIdx->nSample; /* Smallest sample larger than or equal to pRec */ - int iTest; /* Next sample to test */ - int res; /* Result of comparison operation */ - -#ifndef SQLITE_DEBUG - UNUSED_PARAMETER( pParse ); -#endif - assert( pRec!=0 ); - iCol = pRec->nField - 1; - assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && iColnSampleCol ); - do{ - iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); - if( res<0 ){ - iMin = iTest+1; - }else{ - i = iTest; - } - }while( res && iMinnSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) - || pParse->db->mallocFailed ); - }else{ - /* Otherwise, pRec must be smaller than sample $i and larger than - ** sample ($i-1). */ - assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 - || pParse->db->mallocFailed ); - assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); - } -#endif /* ifdef SQLITE_DEBUG */ + u16 eOp = pOne->eOperator | pTwo->eOperator; + sqlite3 *db; /* Database connection (for malloc) */ + Expr *pNew; /* New virtual expression */ + int op; /* Operator for the combined expression */ + int idxNew; /* Index in pWC of the next virtual term */ - /* At this point, aSample[i] is the first sample that is greater than - ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less - ** than pVal. If aSample[i]==pVal, then res==0. - */ - if( res==0 ){ - aStat[0] = aSample[i].anLt[iCol]; - aStat[1] = aSample[i].anEq[iCol]; - }else{ - tRowcnt iLower, iUpper, iGap; - if( i==0 ){ - iLower = 0; - iUpper = aSample[0].anLt[iCol]; - }else{ - i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); - iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; - iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; - } - aStat[1] = pIdx->aAvgEq[iCol]; - if( iLower>=iUpper ){ - iGap = 0; + if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp + && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; + /* If we reach this point, it means the two subterms can be combined */ + if( (eOp & (eOp-1))!=0 ){ + if( eOp & (WO_LT|WO_LE) ){ + eOp = WO_LE; }else{ - iGap = iUpper - iLower; - } - if( roundUp ){ - iGap = (iGap*2)/3; - }else{ - iGap = iGap/3; + assert( eOp & (WO_GT|WO_GE) ); + eOp = WO_GE; } - aStat[0] = iLower + iGap; } + db = pWC->pWInfo->pParse->db; + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + if( pNew==0 ) return; + for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* -** If it is not NULL, pTerm is a term that provides an upper or lower -** bound on a range scan. Without considering pTerm, it is estimated -** that the scan will visit nNew rows. This function returns the number -** estimated to be visited after taking pTerm into account. +** Analyze a term that consists of two or more OR-connected +** subterms. So in: ** -** If the user explicitly specified a likelihood() value for this term, -** then the return value is the likelihood multiplied by the number of -** input rows. Otherwise, this function assumes that an "IS NOT NULL" term -** has a likelihood of 0.50, and any other term a likelihood of 0.25. -*/ -static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ - LogEst nRet = nNew; - if( pTerm ){ - if( pTerm->truthProb<=0 ){ - nRet += pTerm->truthProb; - }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ - nRet -= 20; assert( 20==sqlite3LogEst(4) ); - } - } - return nRet; -} - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** This function is called to estimate the number of rows visited by a -** range-scan on a skip-scan index. For example: +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ ** -** CREATE INDEX i1 ON t1(a, b, c); -** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: ** -** Value pLoop->nOut is currently set to the estimated number of rows -** visited for scanning (a=? AND b=?). This function reduces that estimate -** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple -** times (once for each (a,b) combination that matches a=?) is dealt with -** by the caller. +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object ** -** It does this by scanning through all stat4 samples, comparing values -** extracted from pLower and pUpper with the corresponding column in each -** sample. If L and U are the number of samples found to be less than or -** equal to the values extracted from pLower and pUpper respectively, and -** N is the total number of samples, the pLoop->nOut value is adjusted -** as follows: +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: ** -** nOut = nOut * ( min(U - L, 1) / N ) +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** (F) x>A OR (x=A AND y>=B) ** -** If pLower is NULL, or a value cannot be extracted from the term, L is -** set to zero. If pUpper is NULL, or a value cannot be extracted from it, -** U is set to N. +** CASE 1: ** -** Normally, this function sets *pbDone to 1 before returning. However, -** if no value can be extracted from either pLower or pUpper (and so the -** estimate of the number of rows delivered remains unchanged), *pbDone -** is left as is. +** If all subterms are of the form T.C=expr for some single column of C and +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: ** -** If an error occurs, an SQLite error code is returned. Otherwise, -** SQLITE_OK. -*/ -static int whereRangeSkipScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop, /* Update the .nOut value of this loop */ - int *pbDone /* Set to true if at least one expr. value extracted */ -){ - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - sqlite3 *db = pParse->db; - int nLower = -1; - int nUpper = p->nSample+1; - int rc = SQLITE_OK; - int iCol = p->aiColumn[nEq]; - u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; - CollSeq *pColl; - - sqlite3_value *p1 = 0; /* Value extracted from pLower */ - sqlite3_value *p2 = 0; /* Value extracted from pUpper */ - sqlite3_value *pVal = 0; /* Value extracted from record */ - - pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); - if( pLower ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); - nLower = 0; - } - if( pUpper && rc==SQLITE_OK ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); - nUpper = p2 ? 0 : p->nSample; - } - - if( p1 || p2 ){ - int i; - int nDiff; - for(i=0; rc==SQLITE_OK && inSample; i++){ - rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); - if( rc==SQLITE_OK && p1 ){ - int res = sqlite3MemCompare(p1, pVal, pColl); - if( res>=0 ) nLower++; - } - if( rc==SQLITE_OK && p2 ){ - int res = sqlite3MemCompare(p2, pVal, pColl); - if( res>=0 ) nUpper++; - } - } - nDiff = (nUpper - nLower); - if( nDiff<=0 ) nDiff = 1; - - /* If there is both an upper and lower bound specified, and the - ** comparisons indicate that they are close together, use the fallback - ** method (assume that the scan visits 1/64 of the rows) for estimating - ** the number of rows visited. Otherwise, estimate the number of rows - ** using the method described in the header comment for this function. */ - if( nDiff!=1 || pUpper==0 || pLower==0 ){ - int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); - pLoop->nOut -= nAdjust; - *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", - nLower, nUpper, nAdjust*-1, pLoop->nOut)); - } - - }else{ - assert( *pbDone==0 ); - } - - sqlite3ValueFree(p1); - sqlite3ValueFree(p2); - sqlite3ValueFree(pVal); - - return rc; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - -/* -** This function is used to estimate the number of rows that will be visited -** by scanning an index for a range of values. The range may have an upper -** bound, a lower bound, or both. The WHERE clause terms that set the upper -** and lower bounds are represented by pLower and pUpper respectively. For -** example, assuming that index p is on t1(a): +** x = expr1 OR expr2 = x OR x = expr3 ** -** ... FROM t1 WHERE a > ? AND a < ? ... -** |_____| |_____| -** | | -** pLower pUpper +** then create a new virtual term like this: ** -** If either of the upper or lower bound is not present, then NULL is passed in -** place of the corresponding WhereTerm. +** x IN (expr1,expr2,expr3) ** -** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index -** column subject to the range constraint. Or, equivalently, the number of -** equality constraints optimized by the proposed index scan. For example, -** assuming index p is on t1(a, b), and the SQL query is: +** CASE 2: ** -** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** If there are exactly two disjuncts and one side has x>A and the other side +** has x=A (for the same x and A) then add a new virtual conjunct term to the +** WHERE clause of the form "x>=A". Example: ** -** then nEq is set to 1 (as the range restricted column, b, is the second -** left-most column of the index). Or, if the query is: +** x>A OR (x=A AND y>B) adds: x>=A ** -** ... FROM t1 WHERE a > ? AND a < ? ... +** The added conjunct can sometimes be helpful in query planning. ** -** then nEq is set to 0. +** CASE 3: ** -** When this function is called, *pnOut is set to the sqlite3LogEst() of the -** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, this is the number of -** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range constraints pLower and pUpper. -** -** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be -** used, a single range inequality reduces the search space by a factor of 4. -** and a pair of constraints (x>? AND xindexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. +** +** All examples A through E above satisfy case 3. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 3 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 3 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If none of cases 1, 2, or 3 apply, then leave the eOperator set to +** zero. This term is not useful for search. */ -static int whereRangeScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ ){ - int rc = SQLITE_OK; - int nOut = pLoop->nOut; - LogEst nNew; - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - - if( p->nSample>0 - && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) - ){ - if( nEq==pBuilder->nRecValid ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - u8 aff; + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - */ - tRowcnt iLower; - tRowcnt iUpper; + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. + */ + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + sqlite3WhereClauseInit(pOrWc, pWInfo); + sqlite3WhereSplit(pOrWc, pExpr, TK_OR); + sqlite3WhereExprAnalyze(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); - if( pRec ){ - testcase( pRec->nField!=pBuilder->nRecValid ); - pRec->nField = pBuilder->nRecValid; + /* + ** Compute the set of tables that might satisfy cases 1 or 3. + */ + indexable = ~(Bitmask)0; + chngToIN = ~(Bitmask)0; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); + sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + sqlite3WhereExprAnalyze(pSrc, pAndWC); + pAndWC->pOuter = pWC; + testcase( db->mallocFailed ); + if( !db->mallocFailed ){ + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) ){ + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + } + } + } + indexable &= b; } - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ + }else{ + Bitmask b; + b = sqlite3WhereGetMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pOther->leftCursor); } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + indexable &= b; + if( (pOrTerm->eOperator & WO_EQ)==0 ){ + chngToIN = 0; }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; + chngToIN &= b; } + } + } - assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - assert( p->aSortOrder!=0 ); - if( p->aSortOrder[nEq] ){ - /* The roles of pLower and pUpper are swapped for a DESC index */ - SWAP(WhereTerm*, pLower, pUpper); - } + /* + ** Record the set of tables that satisfy case 3. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + pTerm->eOperator = indexable==0 ? 0 : WO_OR; - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; - pLower = 0; - } + /* For a two-way OR, attempt to implementation case 2. + */ + if( indexable && pOrWc->nTerm==2 ){ + int iOne = 0; + WhereTerm *pOne; + while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ + int iTwo = 0; + WhereTerm *pTwo; + while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ + whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); } + } + } - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); - if( iNewa; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & sqlite3WhereGetMask(&pWInfo->sMaskSet, + pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; } + iColumn = pOrTerm->u.leftColumn; + iCursor = pOrTerm->leftCursor; + break; + } + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==sqlite3WhereGetMask(&pWInfo->sMaskSet, iCursor) ); + break; } + testcase( j==1 ); - pBuilder->pRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); + /* We have found a candidate table and column. Check to see if that + ** table and column is common to every term in the OR clause */ + okToChngToIN = 1; + for(; i>=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.leftColumn!=iColumn ){ + okToChngToIN = 0; }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); - } - if( nNewpExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } } - WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n", - (u32)iLower, (u32)iUpper, nOut)); } - }else{ - int bDone = 0; - rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); - if( bDone ) return rc; } - } -#else - UNUSED_PARAMETER(pParse); - UNUSED_PARAMETER(pBuilder); - assert( pLower || pUpper ); -#endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); - nNew = whereRangeAdjust(pLower, nOut); - nNew = whereRangeAdjust(pUpper, nNew); - - /* TUNING: If there is both an upper and lower limit, assume the range is - ** reduced by an additional 75%. This means that, by default, an open-ended - ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the - ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to - ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; - - nOut -= (pLower!=0) + (pUpper!=0); - if( nNew<10 ) nNew = 10; - if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", - pLoop->nOut, nOut)); - } -#endif - pLoop->nOut = (LogEst)nOut; - return rc; -} - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the number of rows that will be returned based on -** an equality constraint x=VALUE and where that VALUE occurs in -** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available -** for that index. When pExpr==NULL that means the constraint is -** "x IS NULL" instead of "x=VALUE". -** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. -** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. -*/ -static int whereEqualScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - int nEq = pBuilder->pNew->u.btree.nEq; - UnpackedRecord *pRec = pBuilder->pRec; - u8 aff; /* Column affinity */ - int rc; /* Subfunction return code */ - tRowcnt a[2]; /* Statistics */ - int bOk; - - assert( nEq>=1 ); - assert( nEq<=p->nColumn ); - assert( p->aSample!=0 ); - assert( p->nSample>0 ); - assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ - return SQLITE_NOTFOUND; - } + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. + */ + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ - /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() - ** below would return the same value. */ - if( nEq>=p->nColumn ){ - *pnRow = 1; - return SQLITE_OK; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator & WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; + } + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); + }else{ + sqlite3ExprListDelete(db, pList); + } + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ + } } - - aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); - pBuilder->pRec = pRec; - if( rc!=SQLITE_OK ) return rc; - if( bOk==0 ) return SQLITE_NOTFOUND; - pBuilder->nRecValid = nEq; - - whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); - *pnRow = a[1]; - - return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Estimate the number of rows that will be returned based on -** an IN constraint where the right-hand side of the IN operator -** is a list of values. Example: -** -** WHERE x IN (1,2,3,4) -** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. -** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. -*/ -static int whereInScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); - int nRecValid = pBuilder->nRecValid; - int rc = SQLITE_OK; /* Subfunction return code */ - tRowcnt nEst; /* Number of rows for a single term */ - tRowcnt nRowEst = 0; /* New estimate of the number of rows */ - int i; /* Loop counter */ - - assert( p->aSample!=0 ); - for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = nRow0; - rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); - nRowEst += nEst; - pBuilder->nRecValid = nRecValid; - } - - if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; - *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); +** We already know that pExpr is a binary operator where both operands are +** column references. This routine checks to see if pExpr is an equivalence +** relation: +** 1. The SQLITE_Transitive optimization must be enabled +** 2. Must be either an == or an IS operator +** 3. Not originating in the ON clause of an OUTER JOIN +** 4. The affinities of A and B must be compatible +** 5a. Both operands use the same collating sequence OR +** 5b. The overall collating sequence is BINARY +** If this routine returns TRUE, that means that the RHS can be substituted +** for the LHS anyplace else in the WHERE clause where the LHS column occurs. +** This is an optimization. No harm comes from returning 0. But if 1 is +** returned when it should not be, then incorrect answers might result. +*/ +static int termIsEquivalence(Parse *pParse, Expr *pExpr){ + char aff1, aff2; + CollSeq *pColl; + const char *zColl1, *zColl2; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + aff1 = sqlite3ExprAffinity(pExpr->pLeft); + aff2 = sqlite3ExprAffinity(pExpr->pRight); + if( aff1!=aff2 + && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) + ){ + return 0; } - assert( pBuilder->nRecValid==nRecValid ); - return rc; + pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); + if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + /* Since pLeft and pRight are both a column references, their collating + ** sequence should always be defined. */ + zColl1 = ALWAYS(pColl) ? pColl->zName : 0; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); + zColl2 = ALWAYS(pColl) ? pColl->zName : 0; + return sqlite3StrICmp(zColl1, zColl2)==0; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** Disable a term in the WHERE clause. Except, do not disable the term -** if it controls a LEFT OUTER JOIN and it did not originate in the ON -** or USING clause of that join. -** -** Consider the term t2.z='ok' in the following queries: -** -** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' -** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' -** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' -** -** The t2.z='ok' is disabled in the in (2) because it originates -** in the ON clause. The term is disabled in (3) because it is not part -** of a LEFT OUTER JOIN. In (1), the term is not disabled. -** -** Disabling a term causes that term to not be tested in the inner loop -** of the join. Disabling is an optimization. When terms are satisfied -** by indices, we disable them to prevent redundant tests in the inner -** loop. We would get the correct results if nothing were ever disabled, -** but joins might run a little slower. The trick is to disable as much -** as we can without disabling too much. If we disabled in (1), we'd get -** the wrong answer. See ticket #813. +** Recursively walk the expressions of a SELECT statement and generate +** a bitmask indicating which tables are used in that expression +** tree. */ -static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ - if( pTerm - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - && (pLevel->notReady & pTerm->prereqAll)==0 - ){ - pTerm->wtFlags |= TERM_CODED; - if( pTerm->iParent>=0 ){ - WhereTerm *pOther = &pTerm->pWC->a[pTerm->iParent]; - if( (--pOther->nChild)==0 ){ - disableTerm(pLevel, pOther); +static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pEList); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pGroupBy); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pWhere); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); } } + pS = pS->pPrior; } + return mask; } /* -** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor +** number of the table that is indexed and *piColumn to the column number +** of the column that is indexed, or -2 if an expression is being indexed. ** -** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_NONE, then no code gets generated. -** -** This routine makes its own copy of zAff so that the caller is free -** to modify zAff after this routine returns. +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. */ -static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ - Vdbe *v = pParse->pVdbe; - if( zAff==0 ){ - assert( pParse->db->mallocFailed ); - return; - } - assert( v!=0 ); - - /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning - ** and end of the affinity string. - */ - while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ - n--; - base++; - zAff++; - } - while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ - n--; +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + Expr *pExpr, /* An operand of a comparison operator */ + int *piCur, /* Write the referenced table cursor number here */ + int *piColumn /* Write the referenced table column number here */ +){ + Index *pIdx; + int i; + int iCur; + if( pExpr->op==TK_COLUMN ){ + *piCur = pExpr->iTable; + *piColumn = pExpr->iColumn; + return 1; } - - /* Code the OP_Affinity opcode if there is anything left to do. */ - if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=(-2) ) continue; + if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + *piCur = iCur; + *piColumn = -2; + return 1; + } + } } + return 0; } - /* -** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be -** coded. +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. ** -** The current value for the constraint is left in register iReg. +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". ** -** For a constraint of the form X=expr, the expression is evaluated and its -** result is left on the stack. For constraints of the form X IN (...) -** this routine sets up a loop that will iterate over all values of X. +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. */ -static int codeEqualityTerm( - Parse *pParse, /* The parsing context */ - WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* The level of the FROM clause we are working on */ - int iEq, /* Index of the equality term within this level */ - int bRev, /* True for reverse-order IN operations */ - int iTarget /* Attempt to leave results in this register */ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ ){ - Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; - int iReg; /* Register holding results */ - - assert( iTarget>0 ); - if( pX->op==TK_EQ ){ - iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); - }else if( pX->op==TK_ISNULL ){ - iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); -#ifndef SQLITE_OMIT_SUBQUERY - }else{ - int eType; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* uppercase equivalent to lowercase */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } - assert( pX->op==TK_IN ); - iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; + if( db->mallocFailed ){ + return; + } + pTerm = &pWC->a[idxTerm]; + pMaskSet = &pWInfo->sMaskSet; + pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); + }else{ + pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); } - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); + } + prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) ){ + int iCur, iColumn; + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; + if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){ + pTerm->leftCursor = iCur; + pTerm->u.leftColumn = iColumn; + pTerm->eOperator = operatorMask(op) & opMask; } - pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - pIn += pLevel->u.in.nIn - 1; - pIn->iCur = iTab; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; + if( pRight + && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn) + ){ + WhereTerm *pNew; + Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + markTermAsChild(pWC, idxNew, idxTerm); + if( op==TK_IS ) pNew->wtFlags |= TERM_IS; + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + + if( termIsEquivalence(pParse, pDup) ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ - pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); + pDup = pExpr; + pNew = pTerm; } - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); - }else{ - pLevel->u.in.nIn = 0; + exprCommute(pParse, pDup); + pNew->leftCursor = iCur; + pNew->u.leftColumn = iColumn; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } -#endif } - disableTerm(pLevel, pTerm); - return iReg; -} -/* -** Generate code that will evaluate all == and IN constraints for an -** index scan. -** -** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). -** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 -** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two -** constraints are coded. This routine will generate code to evaluate -** a==5 and b IN (1,2,3). The current values for a and b will be stored -** in consecutive registers and the index of the first register is returned. -** -** In the example above nEq==2. But this subroutine works for any value -** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell and -** compute the affinity string. -** -** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints -** are == or IN and are covered by the nEq. nExtraReg is 1 if there is -** an inequality constraint (such as the "c>=5 AND c<10" in the example) that -** occurs after the nEq quality constraints. -** -** This routine allocates a range of nEq+nExtraReg memory cells and returns -** the index of the first memory cell in that range. The code that -** calls this routine will use that memory range to store keys for -** start and termination conditions of the loop. -** key value of the loop. If one or more IN operators appear, then -** this routine allocates an additional nEq memory cells for internal -** use. -** -** Before returning, *pzAff is set to point to a buffer containing a -** copy of the column affinity string of the index allocated using -** sqlite3DbMalloc(). Except, entries in the copy of the string associated -** with equality constraints that use NONE affinity are set to -** SQLITE_AFF_NONE. This is to deal with SQL such as the following: -** -** CREATE TABLE t1(a TEXT PRIMARY KEY, b); -** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; -** -** In the example above, the index on t1(a) has TEXT affinity. But since -** the right hand side of the equality constraint (t2.b) has NONE affinity, -** no conversion should be attempted before using a t2.b value as part of -** a key to search the index. Hence the first byte in the returned affinity -** string in this example would be set to SQLITE_AFF_NONE. -*/ -static int codeAllEqualityTerms( - Parse *pParse, /* Parsing context */ - WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - int bRev, /* Reverse the order of IN operators */ - int nExtraReg, /* Number of extra registers to allocate */ - char **pzAff /* OUT: Set to point to affinity string */ -){ - u16 nEq; /* The number of == or IN constraints to code */ - u16 nSkip; /* Number of left-most columns to skip */ - Vdbe *v = pParse->pVdbe; /* The vm under construction */ - Index *pIdx; /* The index being used for this loop */ - WhereTerm *pTerm; /* A single constraint term */ - WhereLoop *pLoop; /* The WhereLoop object */ - int j; /* Loop counter */ - int regBase; /* Base register */ - int nReg; /* Number of registers to allocate */ - char *zAff; /* Affinity string to return */ +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); + } + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ - /* This module is only called on query plans that use an index. */ - pLoop = pLevel->pWLoop; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; - pIdx = pLoop->u.btree.pIndex; - assert( pIdx!=0 ); +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - /* Figure out how many memory cells we will need then allocate them. +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints + ** + ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". If case is not significant (the default + ** for LIKE) then the lower-bound is made all uppercase and the upper- + ** bound is made all lowercase so that the bounds also work when comparing + ** BLOBs. */ - regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; - pParse->nMem += nReg; + if( pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + const char *zCollSeqName; /* Name of collating sequence */ + const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); + + /* Convert the lower bound to upper-case and the upper bound to + ** lower-case (upper-case is less than lower-case in ASCII) so that + ** the range constraints also work for BLOBs + */ + if( noCase && !pParse->db->mallocFailed ){ + int i; + char c; + pTerm->wtFlags |= TERM_LIKE; + for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ + pStr1->u.zToken[i] = sqlite3Toupper(c); + pStr2->u.zToken[i] = sqlite3Tolower(c); + } + } + + if( !db->mallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; + c = sqlite3UpperToLower[c]; + } + *pC = c + 1; + } + zCollSeqName = noCase ? "NOCASE" : "BINARY"; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), + pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), + pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); + } } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ - if( nSkip ){ - int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); - j = sqlite3VdbeAddOp0(v, OP_Goto); - pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), - iIdxCur, 0, regBase, nSkip); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - sqlite3VdbeJumpHere(v, j); - for(j=0; jaiColumn[j]>=0 ); - VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_MATCH auxiliary term to the constraint set if the + ** current expression is of the form: column MATCH expr. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + if( isMatchOfColumn(pExpr) ){ + int idxNew; + Expr *pRight, *pLeft; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; + + pRight = pExpr->x.pList->a[0].pExpr; + pLeft = pExpr->x.pList->a[1].pExpr; + prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); + prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_MATCH; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; } - } + } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ - /* Evaluate the equality constraints +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* When sqlite_stat3 histogram data is available an operator of the + ** form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** Note that the virtual term must be tagged with TERM_VNULL. */ - assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=nSkip; jaLTerm[j]; - assert( pTerm!=0 ); - /* The following testcase is true for indices with redundant columns. - ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ - testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); - if( r1!=regBase+j ){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); - } + if( pExpr->op==TK_NOTNULL + && pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE_Stat34) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; } - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IN ); - if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ - Expr *pRight = pTerm->pExpr->pRight; - if( sqlite3ExprCanBeNull(pRight) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); - VdbeCoverage(v); - } - if( zAff ){ - if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ - zAff[j] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ - zAff[j] = SQLITE_AFF_NONE; - } - } + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. + */ + pTerm->prereqRight |= extraRight; +} + +/*************************************************************************** +** Routines with file scope above. Interface to the rest of the where.c +** subsystem follows. +***************************************************************************/ + +/* +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: +** +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] +** +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. +** +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. +*/ +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + pWC->op = op; + if( pE2==0 ) return; + if( pE2->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + sqlite3WhereSplit(pWC, pE2->pLeft, op); + sqlite3WhereSplit(pWC, pE2->pRight, op); + } +} + +/* +** Initialize a preallocated WhereClause structure. +*/ +SQLITE_PRIVATE void sqlite3WhereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ +){ + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; +} + +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of sqlite3WhereClauseInit(). +*/ +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); } } - *pzAff = zAff; - return regBase; + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } } -#ifndef SQLITE_OMIT_EXPLAIN + /* -** This routine is a helper for explainIndexRange() below +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask = 0; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = sqlite3WhereGetMask(pMaskSet, p->iTable); + return mask; + } + mask = sqlite3WhereExprUsage(pMaskSet, p->pRight); + mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectUsage(pMaskSet, p->x.pSelect); + }else{ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); + } + return mask; +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= sqlite3WhereExprUsage(pMaskSet, pList->a[i].pExpr); + } + } + return mask; +} + + +/* +** Call exprAnalyze on all terms in a WHERE clause. ** -** pStr holds the text of an expression that we are building up one term -** at a time. This routine adds a new term to the end of the expression. -** Terms are separated by AND so add the "AND" text for second and subsequent -** terms only. +** Note that exprAnalyze() might add new virtual terms onto the +** end of the WHERE clause. We do not want to analyze these new +** virtual terms, so start analyzing at the end and work forward +** so that the added virtual terms are never processed. */ -static void explainAppendTerm( - StrAccum *pStr, /* The text expression being built */ - int iTerm, /* Index of this term. First is zero */ - const char *zColumn, /* Name of the column */ - const char *zOp /* Name of the operator */ +SQLITE_PRIVATE void sqlite3WhereExprAnalyze( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ ){ - if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppendAll(pStr, zColumn); - sqlite3StrAccumAppend(pStr, zOp, 1); - sqlite3StrAccumAppend(pStr, "?", 1); + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } } /* -** Argument pLevel describes a strategy for scanning table pTab. This -** function appends text to pStr that describes the subset of table -** rows scanned by the strategy in the form of an SQL expression. +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. ** -** For example, if the query: +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + struct SrcList_item *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + assert( pArgs!=0 ); + for(j=k=0; jnExpr; j++){ + while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){ k++; } + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; + } + pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pColRef->pTab = pTab; + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); + } +} + +/************** End of whereexpr.c *******************************************/ +/************** Begin file where.c *******************************************/ +/* +** 2001 September 15 ** -** SELECT * FROM t1 WHERE a=1 AND b>2; +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** is run and there is an index on (a, b), then this function returns a -** string similar to: +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** "a=? AND b>?" +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". */ -static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ - Index *pIndex = pLoop->u.btree.pIndex; - u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; - int i, j; - Column *aCol = pTab->aCol; - i16 *aiColumn = pIndex->aiColumn; +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ - if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; - sqlite3StrAccumAppend(pStr, " (", 2); - for(i=0; i=nSkip ){ - explainAppendTerm(pStr, i, z, "="); - }else{ - if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3XPrintf(pStr, 0, "ANY(%s)", z); +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* Test variable that can be set to enable WHERE tracing */ +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +/***/ int sqlite3WhereTrace = 0; +#endif + + +/* +** Return the estimated number of output rows from a WHERE clause +*/ +SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return sqlite3LogEstToInt(pWInfo->nRowOut); +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to +** operate directly on the rowis returned by a WHERE clause. Return +** ONEPASS_SINGLE (1) if the statement can operation directly because only +** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass +** optimization can be used on multiple +** +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. +** +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. +*/ +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); + } +#endif + return pWInfo->eOnePass; +} + +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; } + if( p->rRun<=rRun ) return 0; } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} - j = i; - if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(pStr, i++, z, ">"); +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); + } } - if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(pStr, i, z, "<"); + return 0; +} + +/* +** Create a new mask for cursor iCursor. +** +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. +*/ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aiCur[pScan->iEquiv-1]; + iColumn = pScan->aiColumn[pScan->iEquiv-1]; + if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (iColumn!=XN_EXPR + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + ){ + int j; + for(j=0; jnEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] + ){ + testcase( pTerm->eOperator & WO_IS ); + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv++; } - sqlite3StrAccumAppend(pStr, ")", 1); + return 0; } /* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. */ -static void explainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ ){ -#ifndef SQLITE_DEBUG - if( pParse->explain==2 ) -#endif - { - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ - sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ - int isSearch; /* True for a SEARCH. False for SCAN. */ - WhereLoop *pLoop; /* The controlling WhereLoop object */ - u32 flags; /* Flags that describe this loop */ - char *zMsg; /* Text to add to EQP output */ - StrAccum str; /* EQP output string */ - char zBuf[100]; /* Initial space for EQP output string */ + int j = 0; - pLoop = pLevel->pWLoop; - flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + pScan->pIdxExpr = 0; + if( pIdx ){ + j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + } + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; + } + pScan->opMask = opMask; + pScan->k = 0; + pScan->aiCur[0] = iCur; + pScan->aiColumn[0] = iColumn; + pScan->nEquiv = 1; + pScan->iEquiv = 1; + return whereScanNext(pScan); +} - isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); +/* +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur and is one of +** the WO_xx operator codes specified by the op parameter. +** Return a pointer to the term. Return 0 if not found. +** +** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. +*/ +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +){ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; - sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); - if( pItem->pSelect ){ - sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); - }else{ - sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + op &= WO_EQ|WO_IS; + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ + testcase( p->eOperator & WO_IS ); + return p; + } + if( pResult==0 ) pResult = p; } + p = whereScanNext(&scan); + } + return pResult; +} - if( pItem->zAlias ){ - sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); - } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ - const char *zFmt = 0; - Index *pIdx; +/* +** This function searches pList for an entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; - assert( pLoop->u.btree.pIndex!=0 ); - pIdx = pLoop->u.btree.pIndex; - assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - if( isSearch ){ - zFmt = "PRIMARY KEY"; - } - }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "AUTOMATIC COVERING INDEX"; - }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "COVERING INDEX %s"; - }else{ - zFmt = "INDEX %s"; - } - if( zFmt ){ - sqlite3StrAccumAppend(&str, " USING ", 7); - sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); - explainIndexRange(&str, pLoop, pItem->pTab); - } - }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRange; - if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRange = "(rowid=?)"; - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRange = "(rowid>? AND rowid?)"; - }else{ - assert( flags&WHERE_TOP_LIMIT); - zRange = "(rowidnExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); + if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; } - sqlite3StrAccumAppendAll(&str, " USING INTEGER PRIMARY KEY "); - sqlite3StrAccumAppendAll(&str, zRange); - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", - pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); - } -#endif -#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); - }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); } -#endif - zMsg = sqlite3StrAccumFinish(&str); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); } + + return -1; } -#else -# define explainOneScan(u,v,w,x,y,z) -#endif /* SQLITE_OMIT_EXPLAIN */ +/* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iColnColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return 0; /* Assume an indexed expression can always yield a NULL */ + + } +} /* -** Generate code for the start of the iLevel-th loop in the WHERE clause -** implementation described by pWInfo. +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. +** +** A DISTINCT list is redundant if any subset of the columns in the +** DISTINCT list are collectively unique and individually non-null. */ -static Bitmask codeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - Bitmask notReady /* Which tables are currently available */ +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ ){ - int j, k; /* Loop counters */ - int iCur; /* The VDBE cursor for the table */ - int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ - int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ - WhereLoop *pLoop; /* The WhereLoop object being coded */ - WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ - sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ - int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ - int iReleaseReg = 0; /* Temp register to free before returning */ + Table *pTab; + Index *pIdx; + int i; + int iBase; - pParse = pWInfo->pParse; - v = pParse->pVdbe; - pWC = &pWInfo->sWC; - db = pParse->db; - pLevel = &pWInfo->a[iLevel]; - pLoop = pLevel->pWLoop; - pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - iCur = pTabItem->iCursor; - pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); - bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; - /* Create labels for the "break" and "continue" instructions - ** for the current loop. Jump to addrBrk to break out of a loop. - ** Jump to cont to go immediately to the next iteration of the - ** loop. - ** - ** When there is an IN operator, we also have a "addrNxt" label that - ** means to continue with the next IN value combination. When - ** there are no IN operators in the constraints, the "addrNxt" label - ** is the same as "addrBrk". + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } - /* If this is the right table of a LEFT OUTER JOIN, allocate and - ** initialize a memory cell that records if this table matches any - ** row of the left table of the join. + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ - pLevel->iLeftJoin = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + for(i=0; inKeyCol; i++){ + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; + } + } + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } } - /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); - pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); - VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); - pLevel->op = OP_Goto; - }else + return 0; +} -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 1: The table is a virtual-table. Use the VFilter and VNext - ** to access the data. - */ - int iReg; /* P3 Value for OP_VFilter */ - int addrNotFound; - int nConstraint = pLoop->nLTerm; - sqlite3ExprCachePush(pParse); - iReg = sqlite3GetTempRange(pParse, nConstraint+2); - addrNotFound = pLevel->addrBrk; - for(j=0; jaLTerm[j]; - if( pTerm==0 ) continue; - if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; +/* +** Estimate the logarithm of the input value to base 2. +*/ +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; +} + +/* +** Convert OP_Column opcodes to OP_Copy in previously generated code. +** +** This routine runs over generated VDBE code and translates OP_Column +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, +** then each OP_Rowid is transformed into an instruction to increment the +** value stored in its output register. +*/ +static void translateColumnToCopy( + Vdbe *v, /* The VDBE containing code to translate */ + int iStart, /* Translate from this opcode to the end */ + int iTabCur, /* OP_Column/OP_Rowid references to this table */ + int iRegister, /* The first column is in this register */ + int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ +){ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); + int iEnd = sqlite3VdbeCurrentAddr(v); + for(; iStartp1!=iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + iRegister; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + if( bIncrRowid ){ + /* Increment the value stored in the P2 operand of the OP_Rowid. */ + pOp->opcode = OP_AddImm; + pOp->p1 = pOp->p2; + pOp->p2 = 1; }else{ - sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); - } - } - sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, - pLoop->u.vtab.idxStr, - pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); - VdbeCoverage(v); - pLoop->u.vtab.needFree = 0; - for(j=0; ju.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pLoop->aLTerm[j]); + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; } } - pLevel->op = OP_VNext; - pLevel->p1 = iCur; - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 - ){ - /* Case 2: We can directly reference a single row using an - ** equality comparison against the ROWID field. Or - ** we reference multiple rows using a "rowid IN (...)" - ** construct. - */ - assert( pLoop->u.btree.nEq==1 ); - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - iReleaseReg = ++pParse->nMem; - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); - if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); - addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); - VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); - pLevel->op = OP_Noop; - }else if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 - ){ - /* Case 3: We have an inequality comparison against the ROWID field. - */ - int testOp = OP_Noop; - int start; - int memEndValue = 0; - WhereTerm *pStart, *pEnd; - - assert( omitTable==0 ); - j = 0; - pStart = pEnd = 0; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; - if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; - assert( pStart!=0 || pEnd!=0 ); - if( bRev ){ - pTerm = pStart; - pStart = pEnd; - pEnd = pTerm; - } - if( pStart ){ - Expr *pX; /* The expression that defines the start bound */ - int r1, rTemp; /* Registers for holding the start boundary */ + } +} - /* The following constant maps TK_xx codes into corresponding - ** seek opcodes. It depends on a particular ordering of TK_xx - */ - const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGT, - /* TK_LE */ OP_SeekLE, - /* TK_LT */ OP_SeekLT, - /* TK_GE */ OP_SeekGE - }; - assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ - assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - - assert( (pStart->wtFlags & TERM_VNULL)==0 ); - testcase( pStart->wtFlags & TERM_VIRTUAL ); - pX = pStart->pExpr; - assert( pX!=0 ); - testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); - VdbeComment((v, "pk")); - VdbeCoverageIf(v, pX->op==TK_GT); - VdbeCoverageIf(v, pX->op==TK_LE); - VdbeCoverageIf(v, pX->op==TK_LT); - VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); - sqlite3ReleaseTempReg(pParse, rTemp); - disableTerm(pLevel, pStart); - }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - } - if( pEnd ){ - Expr *pX; - pX = pEnd->pExpr; - assert( pX!=0 ); - assert( (pEnd->wtFlags & TERM_VNULL)==0 ); - testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ - testcase( pEnd->wtFlags & TERM_VIRTUAL ); - memEndValue = ++pParse->nMem; - sqlite3ExprCode(pParse, pX->pRight, memEndValue); - if( pX->op==TK_LT || pX->op==TK_GT ){ - testOp = bRev ? OP_Le : OP_Ge; - }else{ - testOp = bRev ? OP_Lt : OP_Gt; - } - disableTerm(pLevel, pEnd); - } - start = sqlite3VdbeCurrentAddr(v); - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iCur; - pLevel->p2 = start; - assert( pLevel->p5==0 ); - if( testOp!=OP_Noop ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); - VdbeCoverageIf(v, testOp==OP_Le); - VdbeCoverageIf(v, testOp==OP_Lt); - VdbeCoverageIf(v, testOp==OP_Ge); - VdbeCoverageIf(v, testOp==OP_Gt); - sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); - } - }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. - ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: - ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 - ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: - ** - ** x=5 AND z<10 - ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. - ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ - static const u8 aStartOp[] = { - 0, - 0, - OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ - OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ - }; - static const u8 aEndOp[] = { - OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ - OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ - OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ - OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ - }; - u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int regBase; /* Base register holding constraint values */ - WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ - WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ - int startEq; /* True if range start uses ==, >= or <= */ - int endEq; /* True if range end uses ==, >= or <= */ - int start_constraints; /* Start of range is constrained */ - int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ - char *zStartAff; /* Affinity for start of range constraint */ - char cEndAff = 0; /* Affinity for end of range constraint */ - u8 bSeekPastNull = 0; /* True to seek past initial nulls */ - u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ +/* +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) +static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); + } + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); + } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); +} +#else +#define TRACE_IDX_INPUTS(A) +#define TRACE_IDX_OUTPUTS(A) +#endif - pIdx = pLoop->u.btree.pIndex; - iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. +*/ +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + struct SrcList_item *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + testcase( pTerm->pExpr->op==TK_IS ); + return 1; +} +#endif - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->u.btree.nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - /* Find any inequality constraint terms for the start and end - ** of the range. - */ - j = nEq; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = pLoop->aLTerm[j++]; - nExtraReg = 1; - } - if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = pLoop->aLTerm[j++]; - nExtraReg = 1; - if( pRangeStart==0 - && (j = pIdx->aiColumn[nEq])>=0 - && pIdx->pTable->aCol[j].notNull==0 - ){ - bSeekPastNull = 1; - } - } - assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. +*/ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ +){ + int nKeyCol; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ + struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + int addrCounter; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); - /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). - */ - if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) - || (bRev && pIdx->nKeyCol==nEq) - ){ - SWAP(WhereTerm *, pRangeEnd, pRangeStart); - SWAP(u8, bSeekPastNull, bStopAtNull); + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nKeyCol = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; + idxCols = 0; + for(pTerm=pWC->a; pTermpExpr; + assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ + || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ + || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ + if( pLoop->prereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && !ExprHasProperty(pExpr, EP_FromJoin) + && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pExpr, 0)); } - - testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); - testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); - startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); - endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); - start_constraints = pRangeStart || nEq>0; - - /* Seek the index cursor to the start of the range. */ - nConstraint = nEq; - if( pRangeStart ){ - Expr *pRight = pRangeStart->pExpr->pRight; - sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); + if( termCanDriveIndex(pTerm, pSrc, notReady) ){ + int iCol = pTerm->u.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; } - if( zStartAff ){ - if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ - zStartAff[nEq] = SQLITE_AFF_NONE; + if( (idxCols & cMask)==0 ){ + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; } - } - nConstraint++; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); - }else if( bSeekPastNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; - startEq = 0; - start_constraints = 1; - } - codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); - VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); - VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); - VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); - - /* Load the value for the inequality constraint at the end of the - ** range (if any). - */ - nConstraint = nEq; - if( pRangeEnd ){ - Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); - } - if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE - && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) - ){ - codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); - } - nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); - }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; - nConstraint++; - } - sqlite3DbFree(db, zStartAff); - - /* Top of the loop body */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - - /* Check if the index cursor is past the end of the range. */ - if( nConstraint ){ - op = aEndOp[bRev*2 + endEq]; - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); - testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); - testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); - testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); - } - - /* Seek the table cursor, if required */ - disableTerm(pLevel, pRangeStart); - disableTerm(pLevel, pRangeEnd); - if( omitTable ){ - /* pIdx is a covering index. No need to access the main table. */ - }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ - }else if( iCur!=iIdxCur ){ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + pLoop->aLTerm[nKeyCol++] = pTerm; + idxCols |= cMask; } - sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, - iRowidReg, pPk->nKeyCol); VdbeCoverage(v); - } - - /* Record the instruction used to terminate the loop. Disable - ** WHERE clause terms made redundant by the index range scan. - */ - if( pLoop->wsFlags & WHERE_ONEROW ){ - pLevel->op = OP_Noop; - }else if( bRev ){ - pLevel->op = OP_Prev; - }else{ - pLevel->op = OP_Next; - } - pLevel->p1 = iIdxCur; - pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; - if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); } - }else - -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLoop->wsFlags & WHERE_MULTI_OR ){ - /* Case 5: Two or more separately indexed terms connected by OR - ** - ** Example: - ** - ** CREATE TABLE t1(a,b,c,d); - ** CREATE INDEX i1 ON t1(a); - ** CREATE INDEX i2 ON t1(b); - ** CREATE INDEX i3 ON t1(c); - ** - ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) - ** - ** In the example, there are three indexed terms connected by OR. - ** The top of the loop looks like this: - ** - ** Null 1 # Zero the rowset in reg 1 - ** - ** Then, for each indexed term, the following. The arguments to - ** RowSetTest are such that the rowid of the current row is inserted - ** into the RowSet. If it is already present, control skips the - ** Gosub opcode and jumps straight to the code generated by WhereEnd(). - ** - ** sqlite3WhereBegin() - ** RowSetTest # Insert rowid into rowset - ** Gosub 2 A - ** sqlite3WhereEnd() - ** - ** Following the above, code to terminate the loop. Label A, the target - ** of the Gosub above, jumps to the instruction right after the Goto. - ** - ** Null 1 # Zero the rowset in reg 1 - ** Goto B # The loop is finished. - ** - ** A: # Return data, whatever. - ** - ** Return 2 # Jump back to the Gosub - ** - ** B: - ** - ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephemeral index instead of a RowSet to record the primary - ** keys of the rows we have already seen. - ** - */ - WhereClause *pOrWc; /* The OR-clause broken out into subterms */ - SrcList *pOrTab; /* Shortened table list or OR-clause generation */ - Index *pCov = 0; /* Potential covering index (or NULL) */ - int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ + } + assert( nKeyCol>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; - int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ - int regRowset = 0; /* Register for RowSet object */ - int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ - int iRetInit; /* Address of regReturn init */ - int untestedTerms = 0; /* Some terms not completely tested */ - int ii; /* Loop counter */ - u16 wctrlFlags; /* Flags for sub-WHERE clause */ - Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; - - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->eOperator & WO_OR ); - assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); - pOrWc = &pTerm->u.pOrInfo->wc; - pLevel->op = OP_Return; - pLevel->p1 = regReturn; + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = MIN(BMS-1,pTable->nCol); + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; + } - /* Set up a new SrcList in pOrTab containing the table being scanned - ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. - ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). - */ - if( pWInfo->nLevel>1 ){ - int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ - nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); - if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (u8)(nNotReady + 1); - pOrTab->nSrc = pOrTab->nAlloc; - memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); - origSrc = pWInfo->pTabList->a; - for(k=1; k<=nNotReady; k++){ - memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + /* Construct the Index object to describe this index */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + if( pIdx==0 ) goto end_auto_index_create; + pLoop->u.btree.pIndex = pIdx; + pIdx->zName = "auto-index"; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); + if( (idxCols & cMask)==0 ){ + Expr *pX = pTerm->pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.leftColumn; + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->azColl[n] = pColl ? pColl->zName : "BINARY"; + n++; } - }else{ - pOrTab = pWInfo->pTabList; } + } + assert( (u32)n==pLoop->u.btree.nEq ); - /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephemeral index - ** capable of holding primary keys in the case of a WITHOUT ROWID. - ** - ** Also initialize regReturn to contain the address of the instruction - ** immediately following the OP_Return at the bottom of the loop. This - ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to - ** fall through to the next instruction, just as an OP_Next does if - ** called on an uninitialized cursor. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - if( HasRowid(pTab) ){ - regRowset = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - regRowset = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); - sqlite3VdbeSetP4KeyInfo(pParse, pPk); - } - regRowid = ++pParse->nMem; + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; } - iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - - /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z - ** That way, terms in y that are factored into the disjunction will - ** be picked up by the recursive calls to sqlite3WhereBegin() below. - ** - ** Actually, each subexpression is converted to "xN AND w" where w is - ** the "interesting" terms of z - terms that did not originate in the - ** ON or USING clause of a LEFT JOIN, and terms that are usable as - ** indices. - ** - ** This optimization also only applies if the (x1 OR x2 OR ...) term - ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 - */ - if( pWC->nTerm>1 ){ - int iTerm; - for(iTerm=0; iTermnTerm; iTerm++){ - Expr *pExpr = pWC->a[iTerm].pExpr; - if( &pWC->a[iTerm] == pTerm ) continue; - if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; - if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); - } - if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); - } + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; } + } + assert( n==nKeyCol ); + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = "BINARY"; - /* Run a separate WHERE clause for each term of the OR clause. After - ** eliminating duplicates from other WHERE clauses, the action for each - ** sub-WHERE clause is to to invoke the main loop body as a subroutine. - */ - wctrlFlags = WHERE_OMIT_OPEN_CLOSE - | WHERE_FORCE_TABLE - | WHERE_ONETABLE_ONLY; - for(ii=0; iinTerm; ii++){ - WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int j1 = 0; /* Address of jump operation */ - if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ - pAndExpr->pLeft = pOrExpr; - pOrExpr = pAndExpr; - } - /* Loop through table entries that match term pOrTerm. */ - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - wctrlFlags, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); - if( pSubWInfo ){ - WhereLoop *pSubLoop; - explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); - /* This is the sub-WHERE clause body. First skip over - ** duplicate rows from prior sub-WHERE clauses, and record the - ** rowid (or PRIMARY KEY) for the current row so that the same - ** row will be skipped in subsequent sub-WHERE clauses. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); - if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); - VdbeCoverage(v); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - int nPk = pPk->nKeyCol; - int iPk; - - /* Read the PK into an array of temp registers. */ - r = sqlite3GetTempRange(pParse, nPk); - for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); - } - - /* Check if the temp table already contains this key. If so, - ** the row has already been included in the result set and - ** can be ignored (by jumping past the Gosub below). Otherwise, - ** insert the key into the temp table and proceed with processing - ** the row. - ** - ** Use some of the same optimizations as OP_RowSetTest: If iSet - ** is zero, assume that the key cannot already be present in - ** the temp table. And if iSet is -1, assume that there is no - ** need to insert the key into the temp table, as it will never - ** be tested for. */ - if( iSet ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); - VdbeCoverage(v); - } - if( iSet>=0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); - sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); - if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - } - - /* Release the array of temp registers */ - sqlite3ReleaseTempRange(pParse, r, nPk); - } - } - - /* Invoke the main loop body as a subroutine */ - sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); - - /* Jump here (skipping the main loop body subroutine) if the - ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( j1 ) sqlite3VdbeJumpHere(v, j1); - - /* The pSubWInfo->untestedTerms flag means that this OR term - ** contained one or more AND term from a notReady table. The - ** terms from the notReady table could not be tested and will - ** need to be tested later. - */ - if( pSubWInfo->untestedTerms ) untestedTerms = 1; + /* Create the automatic index */ + assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "for %s", pTable->zName)); - /* If all of the OR-connected terms are optimized using the same - ** index, and the index is opened using the same cursor number - ** by each call to sqlite3WhereBegin() made by this loop, it may - ** be possible to use that index as a covering index. - ** - ** If the call to sqlite3WhereBegin() above resulted in a scan that - ** uses an index, and this is either the first OR-connected term - ** processed or the index is the same as that used by all previous - ** terms, set pCov to the candidate covering index. Otherwise, set - ** pCov to NULL to indicate that no candidate covering index will - ** be available. - */ - pSubLoop = pSubWInfo->a[0].pWLoop; - assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 - && (ii==0 || pSubLoop->u.btree.pIndex==pCov) - && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) - ){ - assert( pSubWInfo->a[0].iIdxCur==iCovCur ); - pCov = pSubLoop->u.btree.pIndex; - wctrlFlags |= WHERE_REOPEN_IDX; - }else{ - pCov = 0; - } + /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); + pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + }else{ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + } + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } + regRecord = sqlite3GetTempReg(pParse); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + sqlite3VdbeGoto(v, addrTop); + pTabItem->fg.viaCoroutine = 0; + }else{ + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + } + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); - /* Finish the loop through table entries that match term pOrTerm. */ - sqlite3WhereEnd(pSubWInfo); - } - } - } - pLevel->u.pCovidx = pCov; - if( pCov ) pLevel->iIdxCur = iCovCur; - if( pAndExpr ){ - pAndExpr->pLeft = 0; - sqlite3ExprDelete(db, pAndExpr); - } - sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); - sqlite3VdbeResolveLabel(v, iLoopBody); +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); - if( !untestedTerms ) disableTerm(pLevel, pTerm); - }else -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). +*/ +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + Bitmask mUnusable, /* Ignore terms with these prereqs */ + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; - { - /* Case 6: There is no usable index. We must do a complete - ** scan of the entire table. - */ - static const u8 aStep[] = { OP_Next, OP_Prev }; - static const u8 aStart[] = { OP_Rewind, OP_Last }; - assert( bRev==0 || bRev==1 ); - if( pTabItem->isRecursive ){ - /* Tables marked isRecursive have only a single row that is stored in - ** a pseudo-cursor. No need to Rewind or Next such cursors. */ - pLevel->op = OP_Noop; - }else{ - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - } + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); + nTerm++; } - /* Insert code to test every subexpression that can be completely - ** computed using the current set of tables. + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); - pWInfo->untestedTerms = 1; - continue; + nOrderBy = 0; + if( pOrderBy ){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; + if( i==n){ + nOrderBy = n; } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; } - /* Insert code to test for implied constraints based on transitivity - ** of the "==" operator. - ** - ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" - ** and we are coding the t1 loop and the t2 loop has not yet coded, - ** then we cannot use the "t1.a=t2.b" constraint, but we can code - ** the implied "t1.a=123" constraint. + /* Allocate the sqlite3_index_info structure */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE, *pEAlt; - WhereTerm *pAlt; - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; - if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; - pE = pTerm->pExpr; - assert( !ExprHasProperty(pE, EP_FromJoin) ); - assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); - pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); - if( pAlt==0 ) continue; - if( pAlt->wtFlags & (TERM_CODED) ) continue; - testcase( pAlt->eOperator & WO_EQ ); - testcase( pAlt->eOperator & WO_IN ); - VdbeModuleComment((v, "begin transitive constraint")); - pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); - if( pEAlt ){ - *pEAlt = *pAlt->pExpr; - pEAlt->pLeft = pE->pLeft; - sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); - sqlite3StackFree(db, pEAlt); - } + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); + return 0; } - /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. + /* Initialize the structure. The sqlite3_index_info structure contains + ** many fields that are declared "const" to prevent xBestIndex from + ** changing them. We have to do some funky casting in order to + ** initialize those fields. */ - if( pLevel->iLeftJoin ){ - pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); - VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - assert( pWInfo->untestedTerms ); - continue; - } - assert( pTerm->pExpr ); - sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; - } + pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + *(int*)&pIdxInfo->nConstraint = nTerm; + *(int*)&pIdxInfo->nOrderBy = nOrderBy; + *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; + *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; + *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = + pUsage; + + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); + pIdxCons[j].iColumn = pTerm->u.leftColumn; + pIdxCons[j].iTermOffset = i; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + j++; + } + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; } - return pLevel->notReady; + return pIdxInfo; } -#ifdef WHERETRACE_ENABLED /* -** Print the content of a WhereTerm object +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. +** +** If an error occurs, pParse is populated with an error message and a +** non-zero value is returned. Otherwise, 0 is returned and the output +** part of the sqlite3_index_info structure is left populated. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. */ -static void whereTermPrint(WhereTerm *pTerm, int iTerm){ - if( pTerm==0 ){ - sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); - }else{ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", - iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, - pTerm->eOperator); - sqlite3TreeViewExpr(0, pTerm->pExpr, 0); +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int i; + int rc; + + TRACE_IDX_INPUTS(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + TRACE_IDX_OUTPUTS(p); + + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + pParse->db->mallocFailed = 1; + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } + } + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + + for(i=0; inConstraint; i++){ + if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ + sqlite3ErrorMsg(pParse, + "table %s: xBestIndex returned an invalid plan", pTab->zName); + } } + + return pParse->nErr; } -#endif +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef WHERETRACE_ENABLED +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Print a WhereLoop object for debugging purposes +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: +** +** aStat[0] Est. number of rows less than pRec +** aStat[1] Est. number of rows equal to pRec +** +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. Note that this index is not an index +** into the aSample[] array - it is an index into a virtual set of samples +** based on the contents of aSample[] and the number of fields in record +** pRec. */ -static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+7)/8; - struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); - if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ - if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ +static int whereKeyStats( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ +){ + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int i; /* Index of first sample >= pRec */ + int iSample; /* Smallest sample larger than or equal to pRec */ + int iMin = 0; /* Smallest sample not yet tested */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + int nField; /* Number of fields in pRec */ + tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ + +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); +#endif + assert( pRec!=0 ); + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + + /* Do a binary search to find the first sample greater than or equal + ** to pRec. If pRec contains a single field, the set of samples to search + ** is simply the aSample[] array. If the samples in aSample[] contain more + ** than one fields, all fields following the first are ignored. + ** + ** If pRec contains N fields, where N is more than one, then as well as the + ** samples in aSample[] (truncated to N fields), the search also has to + ** consider prefixes of those samples. For example, if the set of samples + ** in aSample is: + ** + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) + ** aSample[4] = (c, 105) + ** + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** the code actually searches this set: + ** + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) + ** 8: (c, 105) + ** 9: (c, 105) + ** + ** For each sample in the aSample[] array, N samples are present in the + ** effective sample array. In the above, samples 0 and 1 are based on + ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** + ** Often, sample i of each block of N effective samples has (i+1) fields. + ** Except, each sample may be extended to ensure that it is greater than or + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not + ** cause any problems. + */ + nField = pRec->nField; + iCol = 0; + iSample = pIdx->nSample * nField; + do{ + int iSamp; /* Index in aSample[] of test sample */ + int n; /* Number of fields in test sample */ + + iTest = (iMin+iSample)/2; + iSamp = iTest / nField; + if( iSamp>0 ){ + /* The proposed effective sample is a prefix of sample aSample[iSamp]. + ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** fields that is greater than the previous effective sample. */ + for(n=(iTest % nField) + 1; nnField = n; + res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); + if( res<0 ){ + iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; + iMin = iTest+1; + }else if( res==0 && ndb->mallocFailed==0 ){ + if( res==0 ){ + /* If (res==0) is true, then pRec must be equal to sample i. */ + assert( inSample ); + assert( iCol==nField-1 ); + pRec->nField = nField; + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed + ); + }else{ + /* Unless i==pIdx->nSample, indicating that pRec is larger than + ** all samples in the aSample[] array, pRec must be smaller than the + ** (iCol+1) field prefix of sample i. */ + assert( i<=pIdx->nSample && i>=0 ); + pRec->nField = iCol+1; + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); + + /* if i==0 and iCol==0, then record pRec is smaller than all samples + ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must + ** be greater than or equal to the (iCol) field prefix of sample i. + ** If (i>0), then pRec must also be greater than sample (i-1). */ + if( iCol>0 ){ + pRec->nField = iCol; + assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 + || pParse->db->mallocFailed ); + } + if( i>0 ){ + pRec->nField = nField; + assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); + } + } + } +#endif /* ifdef SQLITE_DEBUG */ + + if( res==0 ){ + /* Record pRec is equal to sample i */ + assert( iCol==nField-1 ); + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; + }else{ + /* At this point, the (iCol+1) field prefix of aSample[i] is the first + ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec + ** is larger than all samples in the array. */ + tRowcnt iUpper, iGap; + if( i>=pIdx->nSample ){ + iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + }else{ + iUpper = aSample[i].anLt[iCol]; + } + + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; + aStat[1] = pIdx->aAvgEq[iCol]; + } + + /* Restore the pRec->nField value before returning. */ + pRec->nField = nField; + return i; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return the affinity for a single column of an index. +*/ +static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iColnColumn ); + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + return pIdx->zColAff[iCol]; +} +#endif + + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper +** +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. +** +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: +** +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq is set to 0. +** +** When this function is called, *pnOut is set to the sqlite3LogEst() of the +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, then *pnOut is the number of +** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) +** to account for the range constraints pLower and pUpper. +** +** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be +** used, a single range inequality reduces the search space by a factor of 4. +** and a pair of constraints (x>? AND x123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ +){ + int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + + if( p->nSample>0 && nEqnSampleCol ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values is used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. + */ + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ + + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; + } + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq); + assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER ); + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = p->nRowEst0; + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; + } + + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewwtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } + + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif + pLoop->nOut = (LogEst)nOut; + return rc; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat3 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; + u8 aff; /* Column affinity */ + int rc; /* Subfunction return code */ + tRowcnt a[2]; /* Statistics */ + int bOk; + + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); + assert( p->aSample!=0 ); + assert( p->nSample>0 ); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; + } + + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: +** +** WHERE x IN (1,2,3,4) +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ + + assert( p->aSample!=0 ); + for(i=0; rc==SQLITE_OK && inExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; + } + + if( rc==SQLITE_OK ){ + if( nRowEst > nRow0 ) nRowEst = nRow0; + *pnRow = nRowEst; + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + } + assert( pBuilder->nRecValid==nRecValid ); + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + +#ifdef WHERETRACE_ENABLED +/* +** Print the content of a WhereTerm object +*/ +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf( + "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator, pTerm->wtFlags); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } +} +#endif + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ int i = sqlite3Strlen30(zName) - 1; while( zName[i]!='_' ) i--; zName += i; @@ -117438,7 +123048,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } @@ -117474,7 +123084,6 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ p->u.vtab.idxStr = 0; }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); sqlite3DbFree(db, p->u.btree.pIndex); p->u.btree.pIndex = 0; } @@ -117538,7 +123147,14 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - whereClauseClear(&pWInfo->sWC); + int i; + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + } + sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; @@ -117549,10 +123165,11 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if both of the following are true: +** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y ** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y ** ** By "proper subset" we mean that X uses fewer WHERE clause terms ** than Y and that every WHERE clause term used by X is also used @@ -117560,19 +123177,25 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } + if( pY->nSkip > pX->nSkip ) return 0; if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -117594,33 +123217,24 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -117665,8 +123279,9 @@ static WhereLoop **whereLoopFindLesser( /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. */ + ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 && (p->prereq & pTemplate->prereq)==pTemplate->prereq @@ -117736,18 +123351,20 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** and prereqs. */ if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ #if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = + u16 n = pBuilder->pOrSet->n; + int x = #endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, pTemplate->nOut); #if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } #endif + } return SQLITE_OK; } @@ -117825,10 +123442,30 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. -** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. +** +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -117837,9 +123474,10 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; - int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -117852,20 +123490,27 @@ static void whereLoopOutputAdjust( } if( j<0 ){ if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ pLoop->nOut += pTerm->truthProb; }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&WO_EQ ) nEq++; + if( pTerm->eOperator&(WO_EQ|WO_IS) ){ + Expr *pRight = pTerm->pExpr->pRight; + testcase( pTerm->pExpr->op==TK_IS ); + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut>nRow-10 ){ - pLoop->nOut = nRow - 10; - } + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } /* @@ -117906,10 +123551,9 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ - int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ @@ -117922,65 +123566,26 @@ static int whereLoopAddBtreeIndex( assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; - }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + }else if( /*pProbe->tnum<=0 ||*/ (pSrc->fg.jointype & JT_LEFT)!=0 ){ opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ - opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); assert( pNew->u.btree.nEqnColumn ); - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; - } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -117990,12 +123595,16 @@ static int whereLoopAddBtreeIndex( int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + && indexColumnNotNull(pProbe, saved_nEq) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; + /* Do not allow the upper bound of a LIKE optimization range constraint + ** to mix with a lower range bound from some other source */ + if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; + pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; @@ -118022,10 +123631,14 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - }else if( eOp & (WO_EQ) ){ + }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && !IsUniqueIndex(pProbe) ){ + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==XN_ROWID + || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ + if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; @@ -118039,6 +123652,17 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range contraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + } }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); @@ -118061,10 +123685,10 @@ static int whereLoopAddBtreeIndex( whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ int nEq = ++pNew->u.btree.nEq; - assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && iCol>=0 ){ + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; @@ -118075,12 +123699,12 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; - if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ testcase( eOp & WO_EQ ); + testcase( eOp & WO_IS ); testcase( eOp & WO_ISNULL ); rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); }else{ @@ -118143,10 +123767,45 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -118164,18 +123823,25 @@ static int indexMightHelpWithOrderBy( int iCursor ){ ExprList *pOB; + ExprList *aColExpr; int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ) return 0; - if( pExpr->iTable==iCursor ){ + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jjnKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } + } } } return 0; @@ -118205,8 +123871,17 @@ static Bitmask columnsInIndex(Index *pIdx){ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ int i; WhereTerm *pTerm; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( sqlite3ExprImpliesExpr(pTerm->pExpr, pWhere, iTab) ) return 1; + Expr *pExpr = pTerm->pExpr; + if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) + && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + ){ + return 1; + } } return 0; } @@ -118275,9 +123950,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIndex ){ + if( pSrc->pIBIndex ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIndex; + pProbe = pSrc->pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -118297,7 +123972,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ + if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ sPk.pNext = pFirst; @@ -118309,14 +123984,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ - if( !pBuilder->pOrSet + if( !pBuilder->pOrSet /* Not part of an OR optimization */ + && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIndex==0 - && !pSrc->viaCoroutine - && !pSrc->notIndexed - && HasRowid(pTab) - && !pSrc->isCorrelated - && !pSrc->isRecursive + && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -118325,7 +124000,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -118366,7 +124041,7 @@ static int whereLoopAddBtree( } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -118437,7 +124112,7 @@ static int whereLoopAddBtree( /* If there was an INDEXED BY clause, then only that one index is ** considered. */ - if( pSrc->pIndex ) break; + if( pSrc->pIBIndex ) break; } return rc; } @@ -118446,10 +124121,32 @@ static int whereLoopAddBtree( /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +** +** If there are no LEFT or CROSS JOIN joins in the query, both mExtra and +** mUnusable are set to 0. Otherwise, mExtra is a mask of all FROM clause +** entries that occur before the virtual table in the FROM clause and are +** separated from it by at least one LEFT or CROSS JOIN. Similarly, the +** mUnusable mask contains all FROM clause entries that occur after the +** virtual table and are separated from it by at least one LEFT or +** CROSS JOIN. +** +** For example, if the query were: +** +** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; +** +** then mExtra corresponds to (t1, t2) and mUnusable to (t5, t6). +** +** All the tables in mExtra must be scanned before the current virtual +** table. So any terms for which all prerequisites are satisfied by +** mExtra may be specified as "usable" in all calls to xBestIndex. +** Conversely, all tables in mUnusable must be scanned after the current +** virtual table, so any terms for which the prerequisites overlap with +** mUnusable should always be configured as "not-usable" for xBestIndex. */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra + Bitmask mExtra, /* Tables that must be scanned before this one */ + Bitmask mUnusable /* Tables that must be scanned after this one */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ @@ -118470,6 +124167,7 @@ static int whereLoopAddVirtual( WhereLoop *pNew; int rc = SQLITE_OK; + assert( (mExtra & mUnusable)==0 ); pWInfo = pBuilder->pWInfo; pParse = pWInfo->pParse; db = pParse->db; @@ -118478,7 +124176,7 @@ static int whereLoopAddVirtual( pSrc = &pWInfo->pTabList->a[pNew->iTab]; pTab = pSrc->pTab; assert( IsVirtual(pTab) ); - pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + pIdxInfo = allocateIndexInfo(pParse, pWC, mUnusable, pSrc,pBuilder->pOrderBy); if( pIdxInfo==0 ) return SQLITE_NOMEM; pNew->prereq = 0; pNew->rSetup = 0; @@ -118508,7 +124206,7 @@ static int whereLoopAddVirtual( if( (pTerm->eOperator & WO_IN)!=0 ){ seenIn = 1; } - if( pTerm->prereqRight!=0 ){ + if( (pTerm->prereqRight & ~mExtra)!=0 ){ seenVar = 1; }else if( (pTerm->eOperator & WO_IN)==0 ){ pIdxCons->usable = 1; @@ -118516,7 +124214,7 @@ static int whereLoopAddVirtual( break; case 1: /* Constants with IN operators */ assert( seenIn ); - pIdxCons->usable = (pTerm->prereqRight==0); + pIdxCons->usable = (pTerm->prereqRight & ~mExtra)==0; break; case 2: /* Variables without IN */ assert( seenVar ); @@ -118536,6 +124234,7 @@ static int whereLoopAddVirtual( pIdxInfo->orderByConsumed = 0; pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; @@ -118581,6 +124280,7 @@ static int whereLoopAddVirtual( ** (2) Multiple outputs from a single IN value will not merge ** together. */ pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; } } } @@ -118596,6 +124296,14 @@ static int whereLoopAddVirtual( pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -118615,7 +124323,11 @@ static int whereLoopAddVirtual( ** Add WhereLoop entries to handle OR terms. This works for either ** btrees or virtual tables. */ -static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ +static int whereLoopAddOr( + WhereLoopBuilder *pBuilder, + Bitmask mExtra, + Bitmask mUnusable +){ WhereInfo *pWInfo = pBuilder->pWInfo; WhereClause *pWC; WhereLoop *pNew; @@ -118674,14 +124386,14 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mExtra); + rc = whereLoopAddVirtual(&sSubBuild, mExtra, mUnusable); }else #endif { rc = whereLoopAddBtree(&sSubBuild, mExtra); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(&sSubBuild, mExtra); + rc = whereLoopAddOr(&sSubBuild, mExtra, mUnusable); } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ @@ -118743,33 +124455,43 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ int iTab; SrcList *pTabList = pWInfo->pTabList; struct SrcList_item *pItem; + struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel]; sqlite3 *db = pWInfo->pParse->db; - int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; - u8 priorJoinType = 0; WhereLoop *pNew; + u8 priorJointype = 0; /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; whereLoopInit(pNew); - for(iTab=0, pItem=pTabList->a; iTaba; pItemiTab = iTab; - pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ + /* This condition is true when pItem is the FROM clause term on the + ** right-hand-side of a LEFT or CROSS JOIN. */ mExtra = mPrior; } - priorJoinType = pItem->jointype; + priorJointype = pItem->fg.jointype; if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(pBuilder, mExtra); + struct SrcList_item *p; + for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); + } + } + rc = whereLoopAddVirtual(pBuilder, mExtra, mUnusable); }else{ rc = whereLoopAddBtree(pBuilder, mExtra); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(pBuilder, mExtra); + rc = whereLoopAddOr(pBuilder, mExtra, mUnusable); } mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } + whereLoopClear(db, pNew); return rc; } @@ -118875,10 +124597,10 @@ static i8 wherePathSatisfiesOrderBy( pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; - pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, WO_EQ|WO_ISNULL, 0); + pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL|WO_IS, 0); if( pTerm==0 ) continue; - if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -118887,6 +124609,7 @@ static i8 wherePathSatisfiesOrderBy( if( !pColl ) pColl = db->pDfltColl; z2 = pColl->zName; if( sqlite3StrICmp(z1, z2)!=0 ) continue; + testcase( pTerm->pExpr->op==TK_IS ); } obSat |= MASKBIT(i); } @@ -118902,7 +124625,8 @@ static i8 wherePathSatisfiesOrderBy( nKeyCol = pIndex->nKeyCol; nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); isOrderDistinct = IsUniqueIndex(pIndex); } @@ -118916,8 +124640,8 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 - && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + && pLoop->nSkip==0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ if( i & WO_ISNULL ){ testcase( isOrderDistinct ); @@ -118934,7 +124658,7 @@ static i8 wherePathSatisfiesOrderBy( revIdx = pIndex->aSortOrder[j]; if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - iColumn = -1; + iColumn = XN_ROWID; revIdx = 0; } @@ -118960,9 +124684,15 @@ static i8 wherePathSatisfiesOrderBy( testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=(-1) ){ + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + continue; + } + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -119011,7 +124741,7 @@ static i8 wherePathSatisfiesOrderBy( Bitmask mTerm; if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; - mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); @@ -119193,10 +124923,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Seed the search with a single WherePath containing zero WhereLoops. ** - ** TUNING: Do not let the number of iterations go above 25. If the cost - ** of computing an automatic index is not paid back within the first 25 + ** TUNING: Do not let the number of iterations go above 28. If the cost + ** of computing an automatic index is not paid back within the first 28 ** rows, then do not use the automatic index. */ - aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); + aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); nFrom = 1; assert( aFrom[0].isOrdered==0 ); if( nOrderBy ){ @@ -119370,7 +125100,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iirevMask = pFrom->revLoop; } if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) - && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 ){ Bitmask revMask = 0; int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, @@ -119484,14 +125214,15 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->zIndex ) return 0; + if( pItem->fg.isIndexedBy ) return 0; iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; - pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + pLoop->nSkip = 0; + pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0); if( pTerm ){ + testcase( pTerm->eOperator & WO_IS ); pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; @@ -119500,15 +125231,17 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int opMask; assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; + opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; jnKeyCol; j++){ - pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); if( pTerm==0 ) break; + testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nKeyCol ) continue; @@ -119527,7 +125260,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pLoop->wsFlags ){ pLoop->nOut = (LogEst)1; pWInfo->a[0].pWLoop = pLoop; - pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; @@ -119652,6 +125385,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3 *db; /* Database connection */ int rc; /* Return code */ + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + )); /* Variable initialization */ db = pParse->db; @@ -119707,6 +125444,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; @@ -119721,8 +125459,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(&pWInfo->sWC, pWInfo); - whereSplit(&pWInfo->sWC, pWhere, TK_AND); + sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); + sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -119746,14 +125484,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Assign a bit from the bitmask to every term in the FROM clause. ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally @@ -119762,27 +125498,18 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ for(ii=0; iinSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(ii=0; iinSrc; ii++){ - Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; - } +#ifdef SQLITE_DEBUG + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m==MASKBIT(ii) ); } #endif - /* Analyze all of the subexpressions. Note that exprAnalyze() might - ** add new virtual terms onto the end of the WHERE clause. We do not - ** want to analyze these virtual terms, so start analyzing at the end - ** and work forward so that the added virtual terms are never processed. - */ - exprAnalyzeAll(pTabList, &pWInfo->sWC); - if( db->mallocFailed ){ - goto whereBeginError; - } + /* Analyze all of the subexpressions. */ + sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); + if( db->mallocFailed ) goto whereBeginError; if( wctrlFlags & WHERE_WANT_DISTINCT ){ if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ @@ -119796,10 +125523,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start ***\n")); + WHERETRACE(0xffff,("*** Optimizer Start *** (wctrlFlags: 0x%x)\n", + wctrlFlags)); #if defined(WHERETRACE_ENABLED) - /* Display all terms of the WHERE clause */ - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ int i; for(i=0; inTerm; i++){ whereTermPrint(&sWLB.pWC->a[i], i); @@ -119811,13 +125538,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; - /* Display all of the WhereLoop objects if wheretrace is enabled */ -#ifdef WHERETRACE_ENABLED /* !=0 */ - if( sqlite3WhereTrace ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ WhereLoop *p; int i; - static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ p->cId = zLabel[i%sizeof(zLabel)]; whereLoopPrint(p, sWLB.pWC); @@ -119838,9 +125564,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; } -#ifdef WHERETRACE_ENABLED /* !=0 */ +#ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ - int ii; sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); if( pWInfo->nOBSat>0 ){ sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); @@ -119870,12 +125595,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( && pResultSet!=0 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ - Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); - if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); + } while( pWInfo->nLevel>=2 ){ WhereTerm *pTerm, *pEnd; pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ @@ -119902,21 +125629,25 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. ** The one-pass algorithm only works if the WHERE clause constrains - ** the statement to update a single row. + ** the statement to update or delete a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ - pWInfo->okOnePass = 1; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + if( bOnerow || ( (wctrlFlags & WHERE_ONEPASS_MULTIROW) + && 0==(wsFlags & WHERE_VIRTUALTABLE) + )){ + pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } } } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ - notReady = ~(Bitmask)0; for(ii=0, pLevel=pWInfo->a; iiwsFlags & WHERE_IDX_ONLY)==0 && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = OP_OpenRead; - if( pWInfo->okOnePass ){ + if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; pWInfo->aiCurOnePass[0] = pTabItem->iCursor; }; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); assert( pTabItem->iCursor==pLevel->iTabCur ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nColeOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); + if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed; int n = 0; for(; b; b=b>>1, n++){} @@ -119957,6 +125688,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, + (const u8*)&pTabItem->colUsed, P4_INT64); +#endif }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -119973,7 +125708,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** WITHOUT ROWID table. No need for a separate index */ iIndexCur = pLevel->iTabCur; op = 0; - }else if( pWInfo->okOnePass ){ + }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ Index *pJ = pTabItem->pTab->pIndex; iIndexCur = iIdxCur; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); @@ -119995,11 +125730,31 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( op ){ sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIx); + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ + } VdbeComment((v, "%s", pIx->zName)); +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + { + u64 colUsed = 0; + int ii, jj; + for(ii=0; iinColumn; ii++){ + jj = pIx->aiColumn[ii]; + if( jj<0 ) continue; + if( jj>63 ) jj = 63; + if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; + colUsed |= ((u64)1)<<(ii<63 ? ii : 63); + } + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, + (u8*)&colUsed, P4_INT64); + } +#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ } } if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -120010,7 +125765,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ constructAutomaticIndex(pParse, &pWInfo->sWC, @@ -120018,10 +125776,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + addrExplain = sqlite3WhereExplainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = codeOneLoopStart(pWInfo, ii, notReady); + notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); + } } /* Done. */ @@ -120079,15 +125842,24 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } - sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } + if( pLevel->addrLikeRep ){ + int op; + if( sqlite3VdbeGetOp(v, pLevel->addrLikeRep-1)->p1 ){ + op = OP_DecrJumpZero; + }else{ + op = OP_JumpZeroIncr; + } + sqlite3VdbeAddOp2(v, op, pLevel->iLikeRepCntr, pLevel->addrLikeRep); + VdbeCoverage(v); + } if( pLevel->iLeftJoin ){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 @@ -120101,7 +125873,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLevel->op==OP_Return ){ sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + sqlite3VdbeGoto(v, pLevel->addrFirst); } sqlite3VdbeJumpHere(v, addr); } @@ -120125,26 +125897,12 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; /* For a co-routine, change all OP_Column references to the table of - ** the co-routine into OP_SCopy of result contained in a register. + ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine && !db->mallocFailed ){ - last = sqlite3VdbeCurrentAddr(v); - k = pLevel->addrBody; - pOp = sqlite3VdbeGetOp(v, k); - for(; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - pOp->opcode = OP_Copy; - pOp->p1 = pOp->p2 + pTabItem->regResult; - pOp->p2 = pOp->p3; - pOp->p3 = 0; - }else if( pOp->opcode==OP_Rowid ){ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; - } - } + if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ + translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, + pTabItem->regResult, 0); continue; } @@ -120158,7 +125916,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int ws = pLoop->wsFlags; - if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } if( (ws & WHERE_INDEXED)!=0 @@ -120185,7 +125943,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed ){ + if( pIdx + && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) + && !db->mallocFailed + ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); @@ -120197,6 +125958,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; + assert( x>=0 ); } x = sqlite3ColumnOfIndex(pIdx, x); if( x>=0 ){ @@ -120234,6 +125996,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** in the input grammar file. */ /* #include */ +/* #include "sqliteInt.h" */ /* ** Disable all error recovery processing in the parser push-down @@ -120281,6 +126044,28 @@ struct TrigEvent { int a; IdList * b; }; struct AttachKey { int type; Token key; }; + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + if( p->pPrior ){ + Select *pNext = 0, *pLoop; + int mxSelect, cnt = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } + } + /* This is a utility routine used to set the ExprSpan.zStart and ** ExprSpan.zEnd values of pOut so that the span covers the complete ** range of text beginning with pStart and going to the end of pEnd. @@ -120353,6 +126138,29 @@ struct AttachKey { int type; Token key; }; pOut->zStart = pPreOp->z; pOut->zEnd = pOperand->zEnd; } + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } /* Next is all token values, in a form suitable for use by makeheaders. ** This section will be null unless lemon is run with the -m switch. */ @@ -120397,10 +126205,17 @@ struct AttachKey { int type; Token key; }; ** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ #define YYCODETYPE unsigned char #define YYNOCODE 254 @@ -120433,12 +126248,17 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 642 -#define YYNRULE 327 #define YYFALLBACK 1 -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) +#define YYNSTATE 436 +#define YYNRULE 328 +#define YY_MAX_SHIFT 435 +#define YY_MIN_SHIFTREDUCE 649 +#define YY_MAX_SHIFTREDUCE 976 +#define YY_MIN_REDUCE 977 +#define YY_MAX_REDUCE 1304 +#define YY_ERROR_ACTION 1305 +#define YY_ACCEPT_ACTION 1306 +#define YY_NO_ACTION 1307 /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -120465,16 +126285,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -120504,463 +126328,446 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. */ -#define YY_ACTTAB_COUNT (1497) +#define YY_ACTTAB_COUNT (1501) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, - /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, - /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, - /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, - /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, - /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, - /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, - /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, - /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, - /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, - /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, - /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, - /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, - /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, - /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, - /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, - /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, - /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, - /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, - /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, - /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, - /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, - /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, - /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, - /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, - /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, - /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, - /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, - /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, - /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, - /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, - /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, - /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, - /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, - /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, - /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, - /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, - /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, - /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, - /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, - /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, - /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, - /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, - /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, - /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, - /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, - /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, - /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, - /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, - /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, - /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, - /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, - /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, - /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, - /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, - /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, - /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, - /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, - /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, - /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, - /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, - /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, - /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, - /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, - /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, - /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, - /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, - /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, - /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, - /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, - /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, - /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, - /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, - /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, - /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, - /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, - /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, - /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, - /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, - /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, - /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, - /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, - /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, - /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, - /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, - /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, - /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, - /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, - /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, - /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, - /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, - /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, - /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, - /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, - /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, - /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, - /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, - /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, - /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, - /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, - /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, - /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, - /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, - /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, - /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, - /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, - /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, - /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, - /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, - /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, - /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, - /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, - /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, - /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, - /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, - /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, - /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, - /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, - /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, - /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, - /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, - /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, - /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, - /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, - /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, - /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, - /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, - /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, - /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, - /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, - /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, - /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, - /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, - /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, - /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, - /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, - /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, - /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, - /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, - /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, - /* 1490 */ 971, 971, 971, 971, 971, 971, 338, + /* 0 */ 311, 1306, 145, 651, 2, 192, 652, 338, 780, 92, + /* 10 */ 92, 92, 92, 85, 90, 90, 90, 90, 89, 89, + /* 20 */ 88, 88, 88, 87, 335, 88, 88, 88, 87, 335, + /* 30 */ 327, 856, 856, 92, 92, 92, 92, 776, 90, 90, + /* 40 */ 90, 90, 89, 89, 88, 88, 88, 87, 335, 86, + /* 50 */ 83, 166, 93, 94, 84, 868, 871, 860, 860, 91, + /* 60 */ 91, 92, 92, 92, 92, 335, 90, 90, 90, 90, + /* 70 */ 89, 89, 88, 88, 88, 87, 335, 311, 780, 90, + /* 80 */ 90, 90, 90, 89, 89, 88, 88, 88, 87, 335, + /* 90 */ 123, 808, 689, 689, 689, 689, 112, 230, 430, 257, + /* 100 */ 809, 698, 430, 86, 83, 166, 324, 55, 856, 856, + /* 110 */ 201, 158, 276, 387, 271, 386, 188, 689, 689, 828, + /* 120 */ 833, 49, 944, 269, 833, 49, 123, 87, 335, 93, + /* 130 */ 94, 84, 868, 871, 860, 860, 91, 91, 92, 92, + /* 140 */ 92, 92, 342, 90, 90, 90, 90, 89, 89, 88, + /* 150 */ 88, 88, 87, 335, 311, 328, 333, 332, 701, 408, + /* 160 */ 394, 69, 690, 691, 690, 691, 715, 910, 251, 354, + /* 170 */ 250, 698, 704, 430, 908, 430, 909, 89, 89, 88, + /* 180 */ 88, 88, 87, 335, 391, 856, 856, 690, 691, 183, + /* 190 */ 95, 340, 384, 381, 380, 833, 31, 833, 49, 912, + /* 200 */ 912, 333, 332, 379, 123, 311, 93, 94, 84, 868, + /* 210 */ 871, 860, 860, 91, 91, 92, 92, 92, 92, 114, + /* 220 */ 90, 90, 90, 90, 89, 89, 88, 88, 88, 87, + /* 230 */ 335, 430, 408, 399, 435, 657, 856, 856, 346, 57, + /* 240 */ 232, 828, 109, 20, 912, 912, 231, 393, 937, 760, + /* 250 */ 97, 751, 752, 833, 49, 708, 708, 93, 94, 84, + /* 260 */ 868, 871, 860, 860, 91, 91, 92, 92, 92, 92, + /* 270 */ 707, 90, 90, 90, 90, 89, 89, 88, 88, 88, + /* 280 */ 87, 335, 311, 114, 22, 706, 688, 58, 408, 390, + /* 290 */ 251, 349, 240, 749, 752, 689, 689, 847, 685, 115, + /* 300 */ 21, 231, 393, 689, 689, 697, 183, 355, 430, 384, + /* 310 */ 381, 380, 192, 856, 856, 780, 123, 160, 159, 223, + /* 320 */ 379, 738, 25, 315, 362, 841, 143, 689, 689, 835, + /* 330 */ 833, 48, 339, 937, 93, 94, 84, 868, 871, 860, + /* 340 */ 860, 91, 91, 92, 92, 92, 92, 914, 90, 90, + /* 350 */ 90, 90, 89, 89, 88, 88, 88, 87, 335, 311, + /* 360 */ 840, 840, 840, 266, 430, 690, 691, 778, 114, 1300, + /* 370 */ 1300, 430, 1, 690, 691, 697, 688, 689, 689, 689, + /* 380 */ 689, 689, 689, 287, 298, 780, 833, 10, 686, 115, + /* 390 */ 856, 856, 355, 833, 10, 828, 366, 690, 691, 363, + /* 400 */ 321, 76, 123, 74, 23, 737, 807, 323, 356, 353, + /* 410 */ 847, 93, 94, 84, 868, 871, 860, 860, 91, 91, + /* 420 */ 92, 92, 92, 92, 940, 90, 90, 90, 90, 89, + /* 430 */ 89, 88, 88, 88, 87, 335, 311, 806, 841, 429, + /* 440 */ 713, 941, 835, 430, 251, 354, 250, 690, 691, 690, + /* 450 */ 691, 690, 691, 86, 83, 166, 24, 942, 151, 753, + /* 460 */ 285, 907, 403, 907, 164, 833, 10, 856, 856, 965, + /* 470 */ 306, 754, 679, 840, 840, 840, 795, 216, 794, 222, + /* 480 */ 906, 344, 906, 904, 86, 83, 166, 286, 93, 94, + /* 490 */ 84, 868, 871, 860, 860, 91, 91, 92, 92, 92, + /* 500 */ 92, 430, 90, 90, 90, 90, 89, 89, 88, 88, + /* 510 */ 88, 87, 335, 311, 430, 724, 352, 705, 427, 699, + /* 520 */ 700, 376, 210, 833, 49, 793, 397, 857, 857, 940, + /* 530 */ 213, 762, 727, 334, 699, 700, 833, 10, 86, 83, + /* 540 */ 166, 345, 396, 902, 856, 856, 941, 385, 833, 9, + /* 550 */ 406, 869, 872, 187, 890, 728, 347, 398, 404, 977, + /* 560 */ 652, 338, 942, 954, 413, 93, 94, 84, 868, 871, + /* 570 */ 860, 860, 91, 91, 92, 92, 92, 92, 861, 90, + /* 580 */ 90, 90, 90, 89, 89, 88, 88, 88, 87, 335, + /* 590 */ 311, 1219, 114, 430, 834, 430, 5, 165, 192, 688, + /* 600 */ 832, 780, 430, 723, 430, 234, 325, 189, 163, 316, + /* 610 */ 356, 955, 115, 235, 269, 833, 35, 833, 36, 747, + /* 620 */ 720, 856, 856, 793, 833, 12, 833, 27, 745, 174, + /* 630 */ 968, 1290, 968, 1291, 1290, 310, 1291, 693, 317, 245, + /* 640 */ 264, 311, 93, 94, 84, 868, 871, 860, 860, 91, + /* 650 */ 91, 92, 92, 92, 92, 832, 90, 90, 90, 90, + /* 660 */ 89, 89, 88, 88, 88, 87, 335, 430, 320, 213, + /* 670 */ 762, 780, 856, 856, 920, 920, 369, 257, 966, 220, + /* 680 */ 966, 396, 663, 664, 665, 242, 259, 244, 262, 833, + /* 690 */ 37, 650, 2, 93, 94, 84, 868, 871, 860, 860, + /* 700 */ 91, 91, 92, 92, 92, 92, 430, 90, 90, 90, + /* 710 */ 90, 89, 89, 88, 88, 88, 87, 335, 311, 430, + /* 720 */ 239, 430, 917, 368, 430, 238, 916, 793, 833, 38, + /* 730 */ 430, 825, 430, 66, 430, 392, 430, 766, 766, 430, + /* 740 */ 367, 833, 39, 833, 28, 430, 833, 29, 68, 856, + /* 750 */ 856, 900, 833, 40, 833, 41, 833, 42, 833, 11, + /* 760 */ 72, 833, 43, 243, 305, 970, 114, 833, 99, 961, + /* 770 */ 93, 94, 84, 868, 871, 860, 860, 91, 91, 92, + /* 780 */ 92, 92, 92, 430, 90, 90, 90, 90, 89, 89, + /* 790 */ 88, 88, 88, 87, 335, 311, 430, 361, 430, 165, + /* 800 */ 147, 430, 186, 185, 184, 833, 44, 430, 289, 430, + /* 810 */ 246, 430, 971, 430, 212, 163, 430, 357, 833, 45, + /* 820 */ 833, 32, 932, 833, 46, 793, 856, 856, 718, 833, + /* 830 */ 47, 833, 33, 833, 117, 833, 118, 75, 833, 119, + /* 840 */ 288, 305, 967, 214, 935, 322, 311, 93, 94, 84, + /* 850 */ 868, 871, 860, 860, 91, 91, 92, 92, 92, 92, + /* 860 */ 430, 90, 90, 90, 90, 89, 89, 88, 88, 88, + /* 870 */ 87, 335, 430, 832, 426, 317, 288, 856, 856, 114, + /* 880 */ 763, 257, 833, 53, 930, 219, 364, 257, 257, 971, + /* 890 */ 361, 396, 257, 257, 833, 34, 257, 311, 93, 94, + /* 900 */ 84, 868, 871, 860, 860, 91, 91, 92, 92, 92, + /* 910 */ 92, 430, 90, 90, 90, 90, 89, 89, 88, 88, + /* 920 */ 88, 87, 335, 430, 217, 318, 124, 253, 856, 856, + /* 930 */ 218, 943, 257, 833, 100, 898, 759, 774, 361, 755, + /* 940 */ 423, 329, 758, 1017, 289, 833, 50, 682, 311, 93, + /* 950 */ 82, 84, 868, 871, 860, 860, 91, 91, 92, 92, + /* 960 */ 92, 92, 430, 90, 90, 90, 90, 89, 89, 88, + /* 970 */ 88, 88, 87, 335, 430, 256, 419, 114, 249, 856, + /* 980 */ 856, 331, 114, 400, 833, 101, 359, 187, 1064, 726, + /* 990 */ 725, 739, 401, 416, 420, 360, 833, 102, 424, 311, + /* 1000 */ 258, 94, 84, 868, 871, 860, 860, 91, 91, 92, + /* 1010 */ 92, 92, 92, 430, 90, 90, 90, 90, 89, 89, + /* 1020 */ 88, 88, 88, 87, 335, 430, 221, 261, 114, 114, + /* 1030 */ 856, 856, 808, 114, 156, 833, 98, 772, 733, 734, + /* 1040 */ 275, 809, 771, 316, 263, 265, 960, 833, 116, 307, + /* 1050 */ 741, 274, 722, 84, 868, 871, 860, 860, 91, 91, + /* 1060 */ 92, 92, 92, 92, 430, 90, 90, 90, 90, 89, + /* 1070 */ 89, 88, 88, 88, 87, 335, 80, 425, 830, 3, + /* 1080 */ 1214, 191, 430, 721, 336, 336, 833, 113, 252, 80, + /* 1090 */ 425, 68, 3, 913, 913, 428, 270, 336, 336, 430, + /* 1100 */ 377, 784, 430, 197, 833, 106, 430, 716, 428, 430, + /* 1110 */ 267, 430, 897, 68, 414, 430, 769, 409, 430, 71, + /* 1120 */ 430, 833, 105, 123, 833, 103, 847, 414, 833, 49, + /* 1130 */ 843, 833, 104, 833, 52, 800, 123, 833, 54, 847, + /* 1140 */ 833, 51, 833, 26, 831, 802, 77, 78, 191, 389, + /* 1150 */ 430, 372, 114, 79, 432, 431, 911, 911, 835, 77, + /* 1160 */ 78, 779, 893, 408, 410, 197, 79, 432, 431, 791, + /* 1170 */ 226, 835, 833, 30, 772, 80, 425, 716, 3, 771, + /* 1180 */ 411, 412, 897, 336, 336, 290, 291, 839, 703, 840, + /* 1190 */ 840, 840, 842, 19, 428, 695, 684, 672, 111, 671, + /* 1200 */ 843, 673, 840, 840, 840, 842, 19, 207, 661, 278, + /* 1210 */ 148, 304, 280, 414, 282, 6, 822, 348, 248, 241, + /* 1220 */ 358, 934, 720, 80, 425, 847, 3, 161, 382, 273, + /* 1230 */ 284, 336, 336, 415, 296, 958, 895, 894, 157, 674, + /* 1240 */ 107, 194, 428, 948, 135, 77, 78, 777, 953, 951, + /* 1250 */ 56, 319, 79, 432, 431, 121, 66, 835, 59, 128, + /* 1260 */ 146, 414, 350, 130, 351, 819, 131, 132, 133, 375, + /* 1270 */ 173, 149, 138, 847, 936, 365, 178, 70, 425, 827, + /* 1280 */ 3, 889, 62, 371, 915, 336, 336, 792, 840, 840, + /* 1290 */ 840, 842, 19, 77, 78, 208, 428, 144, 179, 373, + /* 1300 */ 79, 432, 431, 255, 180, 835, 260, 675, 181, 308, + /* 1310 */ 388, 744, 326, 743, 742, 414, 731, 718, 712, 402, + /* 1320 */ 309, 711, 788, 65, 277, 272, 789, 847, 730, 710, + /* 1330 */ 709, 279, 193, 787, 281, 876, 840, 840, 840, 842, + /* 1340 */ 19, 786, 283, 73, 418, 330, 422, 77, 78, 227, + /* 1350 */ 96, 407, 67, 405, 79, 432, 431, 292, 228, 835, + /* 1360 */ 215, 202, 229, 293, 767, 303, 302, 301, 204, 299, + /* 1370 */ 294, 295, 676, 7, 681, 433, 669, 206, 110, 224, + /* 1380 */ 203, 205, 434, 667, 666, 658, 120, 168, 656, 237, + /* 1390 */ 840, 840, 840, 842, 19, 337, 155, 233, 236, 341, + /* 1400 */ 167, 905, 108, 313, 903, 826, 314, 125, 126, 127, + /* 1410 */ 129, 170, 247, 756, 172, 928, 134, 136, 171, 60, + /* 1420 */ 61, 123, 169, 137, 175, 933, 176, 927, 8, 13, + /* 1430 */ 177, 254, 191, 918, 139, 370, 924, 140, 678, 150, + /* 1440 */ 374, 274, 182, 378, 141, 122, 63, 14, 383, 729, + /* 1450 */ 268, 15, 64, 225, 846, 845, 874, 16, 765, 770, + /* 1460 */ 4, 162, 209, 395, 211, 142, 878, 796, 801, 312, + /* 1470 */ 190, 71, 68, 875, 873, 939, 199, 938, 17, 195, + /* 1480 */ 18, 196, 417, 975, 152, 653, 976, 198, 153, 421, + /* 1490 */ 877, 154, 200, 844, 696, 81, 343, 297, 1019, 1018, + /* 1500 */ 300, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 0 */ 19, 144, 145, 146, 147, 24, 1, 2, 27, 80, /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, - /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, - /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, - /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, - /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, - /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, - /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, - /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, - /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 20 */ 91, 92, 93, 94, 95, 91, 92, 93, 94, 95, + /* 30 */ 19, 50, 51, 80, 81, 82, 83, 212, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 224, + /* 50 */ 225, 226, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 95, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 85, + /* 80 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 90 */ 66, 33, 27, 28, 27, 28, 22, 201, 152, 152, + /* 100 */ 42, 27, 152, 224, 225, 226, 95, 211, 50, 51, + /* 110 */ 99, 100, 101, 102, 103, 104, 105, 27, 28, 59, + /* 120 */ 174, 175, 243, 112, 174, 175, 66, 94, 95, 71, /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, - /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, - /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, - /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, - /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, - /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, - /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, - /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 140 */ 82, 83, 195, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 209, 89, 90, 173, 209, + /* 160 */ 210, 26, 97, 98, 97, 98, 181, 100, 108, 109, + /* 170 */ 110, 97, 174, 152, 107, 152, 109, 89, 90, 91, + /* 180 */ 92, 93, 94, 95, 163, 50, 51, 97, 98, 99, + /* 190 */ 55, 244, 102, 103, 104, 174, 175, 174, 175, 132, + /* 200 */ 133, 89, 90, 113, 66, 19, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 198, /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, - /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, - /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, - /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, - /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, - /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, - /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, - /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, - /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, - /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, - /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 230 */ 95, 152, 209, 210, 148, 149, 50, 51, 100, 53, + /* 240 */ 154, 59, 156, 22, 132, 133, 119, 120, 163, 163, + /* 250 */ 22, 192, 193, 174, 175, 27, 28, 71, 72, 73, + /* 260 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 270 */ 174, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 280 */ 94, 95, 19, 198, 198, 174, 152, 24, 209, 210, + /* 290 */ 108, 109, 110, 192, 193, 27, 28, 69, 164, 165, + /* 300 */ 79, 119, 120, 27, 28, 27, 99, 222, 152, 102, + /* 310 */ 103, 104, 24, 50, 51, 27, 66, 89, 90, 185, + /* 320 */ 113, 187, 22, 157, 239, 97, 58, 27, 28, 101, + /* 330 */ 174, 175, 246, 163, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 11, 85, 86, /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, - /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, - /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, - /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, - /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, - /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, - /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, - /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, - /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, - /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, - /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, - /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, - /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, - /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 360 */ 132, 133, 134, 23, 152, 97, 98, 91, 198, 119, + /* 370 */ 120, 152, 22, 97, 98, 97, 152, 27, 28, 27, + /* 380 */ 28, 27, 28, 227, 160, 97, 174, 175, 164, 165, + /* 390 */ 50, 51, 222, 174, 175, 59, 230, 97, 98, 233, + /* 400 */ 188, 137, 66, 139, 234, 187, 177, 188, 152, 239, + /* 410 */ 69, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 12, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 177, 97, 152, + /* 440 */ 23, 29, 101, 152, 108, 109, 110, 97, 98, 97, + /* 450 */ 98, 97, 98, 224, 225, 226, 22, 45, 24, 47, + /* 460 */ 152, 152, 152, 152, 152, 174, 175, 50, 51, 249, + /* 470 */ 250, 59, 21, 132, 133, 134, 124, 221, 124, 188, + /* 480 */ 171, 172, 171, 172, 224, 225, 226, 152, 71, 72, /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, - /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, - /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, - /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, - /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, - /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, - /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, - /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 500 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 152, 183, 65, 23, 170, 171, + /* 520 */ 172, 19, 23, 174, 175, 26, 152, 50, 51, 12, + /* 530 */ 196, 197, 37, 170, 171, 172, 174, 175, 224, 225, + /* 540 */ 226, 232, 208, 232, 50, 51, 29, 52, 174, 175, + /* 550 */ 188, 74, 75, 51, 103, 60, 222, 163, 209, 0, + /* 560 */ 1, 2, 45, 152, 47, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 101, 85, /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, - /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, - /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, - /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, - /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, - /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 590 */ 19, 140, 198, 152, 23, 152, 22, 98, 24, 152, + /* 600 */ 152, 27, 152, 183, 152, 152, 111, 213, 214, 107, + /* 610 */ 152, 164, 165, 152, 112, 174, 175, 174, 175, 181, + /* 620 */ 182, 50, 51, 124, 174, 175, 174, 175, 190, 26, + /* 630 */ 22, 23, 22, 23, 26, 166, 26, 168, 169, 16, + /* 640 */ 16, 19, 71, 72, 73, 74, 75, 76, 77, 78, /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, - /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, - /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, - /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, - /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, - /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, - /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, - /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, - /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, - /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, - /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, - /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, - /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, - /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, - /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, - /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, - /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 152, 220, 196, + /* 670 */ 197, 97, 50, 51, 108, 109, 110, 152, 70, 221, + /* 680 */ 70, 208, 7, 8, 9, 62, 62, 64, 64, 174, + /* 690 */ 175, 146, 147, 71, 72, 73, 74, 75, 76, 77, + /* 700 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, + /* 710 */ 88, 89, 90, 91, 92, 93, 94, 95, 19, 152, + /* 720 */ 195, 152, 31, 220, 152, 152, 35, 26, 174, 175, + /* 730 */ 152, 163, 152, 130, 152, 115, 152, 117, 118, 152, + /* 740 */ 49, 174, 175, 174, 175, 152, 174, 175, 26, 50, + /* 750 */ 51, 152, 174, 175, 174, 175, 174, 175, 174, 175, + /* 760 */ 138, 174, 175, 140, 22, 23, 198, 174, 175, 152, + /* 770 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 780 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 790 */ 91, 92, 93, 94, 95, 19, 152, 152, 152, 98, + /* 800 */ 24, 152, 108, 109, 110, 174, 175, 152, 152, 152, + /* 810 */ 152, 152, 70, 152, 213, 214, 152, 152, 174, 175, + /* 820 */ 174, 175, 152, 174, 175, 124, 50, 51, 106, 174, + /* 830 */ 175, 174, 175, 174, 175, 174, 175, 138, 174, 175, + /* 840 */ 152, 22, 23, 22, 163, 189, 19, 71, 72, 73, /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, - /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, - /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 870 */ 94, 95, 152, 152, 168, 169, 152, 50, 51, 198, + /* 880 */ 197, 152, 174, 175, 152, 240, 152, 152, 152, 70, + /* 890 */ 152, 208, 152, 152, 174, 175, 152, 19, 71, 72, /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, - /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, - /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, - /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 920 */ 93, 94, 95, 152, 195, 247, 248, 152, 50, 51, + /* 930 */ 195, 195, 152, 174, 175, 195, 195, 26, 152, 195, + /* 940 */ 252, 220, 163, 122, 152, 174, 175, 163, 19, 71, /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, - /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, - /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, + /* 970 */ 92, 93, 94, 95, 152, 195, 252, 198, 240, 50, + /* 980 */ 51, 189, 198, 19, 174, 175, 19, 51, 23, 100, + /* 990 */ 101, 26, 28, 163, 163, 28, 174, 175, 163, 19, /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, - /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, - /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, - /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, - /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1020 */ 91, 92, 93, 94, 95, 152, 240, 152, 198, 198, + /* 1030 */ 50, 51, 33, 198, 123, 174, 175, 116, 7, 8, + /* 1040 */ 101, 42, 121, 107, 152, 152, 23, 174, 175, 26, + /* 1050 */ 152, 112, 183, 73, 74, 75, 76, 77, 78, 79, /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, - /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, - /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, - /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, - /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, - /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, - /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, - /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, - /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, - /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, - /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, - /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, - /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, - /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, - /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, - /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, - /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, - /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, - /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, - /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, - /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, - /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, - /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, - /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, - /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, - /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, - /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, - /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, - /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, - /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, - /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, - /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, - /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, - /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, - /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, - /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, - /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, - /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, - /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, - /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, - /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, - /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, - /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, - /* 1490 */ 253, 253, 253, 253, 253, 253, 141, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 23, 22, + /* 1080 */ 23, 26, 152, 152, 27, 28, 174, 175, 23, 19, + /* 1090 */ 20, 26, 22, 132, 133, 38, 152, 27, 28, 152, + /* 1100 */ 23, 215, 152, 26, 174, 175, 152, 27, 38, 152, + /* 1110 */ 23, 152, 27, 26, 57, 152, 23, 163, 152, 26, + /* 1120 */ 152, 174, 175, 66, 174, 175, 69, 57, 174, 175, + /* 1130 */ 27, 174, 175, 174, 175, 152, 66, 174, 175, 69, + /* 1140 */ 174, 175, 174, 175, 152, 23, 89, 90, 26, 91, + /* 1150 */ 152, 236, 198, 96, 97, 98, 132, 133, 101, 89, + /* 1160 */ 90, 152, 23, 209, 210, 26, 96, 97, 98, 152, + /* 1170 */ 212, 101, 174, 175, 116, 19, 20, 97, 22, 121, + /* 1180 */ 152, 193, 97, 27, 28, 152, 152, 152, 152, 132, + /* 1190 */ 133, 134, 135, 136, 38, 23, 152, 152, 26, 152, + /* 1200 */ 97, 152, 132, 133, 134, 135, 136, 235, 152, 212, + /* 1210 */ 199, 150, 212, 57, 212, 200, 203, 216, 241, 216, + /* 1220 */ 241, 203, 182, 19, 20, 69, 22, 186, 178, 177, + /* 1230 */ 216, 27, 28, 229, 202, 39, 177, 177, 200, 155, + /* 1240 */ 245, 122, 38, 41, 22, 89, 90, 91, 159, 159, + /* 1250 */ 242, 159, 96, 97, 98, 71, 130, 101, 242, 191, + /* 1260 */ 223, 57, 18, 194, 159, 203, 194, 194, 194, 18, + /* 1270 */ 158, 223, 191, 69, 203, 159, 158, 19, 20, 191, + /* 1280 */ 22, 203, 137, 46, 238, 27, 28, 159, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 159, 38, 22, 158, 179, + /* 1300 */ 96, 97, 98, 237, 158, 101, 159, 159, 158, 179, + /* 1310 */ 107, 176, 48, 176, 176, 57, 184, 106, 176, 125, + /* 1320 */ 179, 178, 218, 107, 217, 176, 218, 69, 184, 176, + /* 1330 */ 176, 217, 159, 218, 217, 159, 132, 133, 134, 135, + /* 1340 */ 136, 218, 217, 137, 179, 95, 179, 89, 90, 228, + /* 1350 */ 129, 126, 128, 127, 96, 97, 98, 206, 231, 101, + /* 1360 */ 5, 25, 231, 205, 207, 10, 11, 12, 13, 14, + /* 1370 */ 204, 203, 17, 26, 162, 161, 13, 6, 180, 180, + /* 1380 */ 153, 153, 151, 151, 151, 151, 167, 32, 4, 34, + /* 1390 */ 132, 133, 134, 135, 136, 3, 22, 142, 43, 68, + /* 1400 */ 15, 23, 16, 251, 23, 120, 251, 248, 131, 111, + /* 1410 */ 123, 56, 16, 20, 125, 1, 123, 131, 63, 79, + /* 1420 */ 79, 66, 67, 111, 36, 28, 122, 1, 5, 22, + /* 1430 */ 107, 140, 26, 54, 54, 44, 61, 107, 20, 24, + /* 1440 */ 19, 112, 105, 53, 22, 40, 22, 22, 53, 30, + /* 1450 */ 23, 22, 22, 53, 23, 23, 23, 22, 116, 23, + /* 1460 */ 22, 122, 23, 26, 23, 22, 11, 124, 28, 114, + /* 1470 */ 36, 26, 26, 23, 23, 23, 122, 23, 36, 26, + /* 1480 */ 36, 22, 24, 23, 22, 1, 23, 26, 22, 24, + /* 1490 */ 23, 22, 122, 23, 23, 22, 141, 23, 122, 122, + /* 1500 */ 15, }; -#define YY_SHIFT_USE_DFLT (-86) -#define YY_SHIFT_COUNT (429) -#define YY_SHIFT_MIN (-85) -#define YY_SHIFT_MAX (1383) +#define YY_SHIFT_USE_DFLT (-72) +#define YY_SHIFT_COUNT (435) +#define YY_SHIFT_MIN (-71) +#define YY_SHIFT_MAX (1485) static const short yy_shift_ofst[] = { - /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, - /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, - /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, - /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, - /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, - /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 0 */ 5, 1057, 1355, 1070, 1204, 1204, 1204, 90, 60, -19, + /* 10 */ 58, 58, 186, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 20 */ 67, 67, 182, 336, 65, 250, 135, 263, 340, 417, + /* 30 */ 494, 571, 622, 699, 776, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 50 */ 878, 827, 929, 980, 980, 1156, 1204, 1204, 1204, 1204, /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, - /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, - /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, - /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, - /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, - /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, - /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, - /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, - /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, - /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, - /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, - /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, - /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, - /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, - /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, - /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, - /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, - /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, - /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, - /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, - /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, - /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, - /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, - /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, - /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, - /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, - /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, - /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, - /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, - /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, - /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, - /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, + /* 80 */ 1204, 1204, 1204, 1204, 1258, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, -71, -47, + /* 100 */ -47, -47, -47, -47, -6, 88, -66, 65, 65, 451, + /* 110 */ 502, 112, 112, 33, 127, 278, -30, -72, -72, -72, + /* 120 */ 11, 412, 412, 268, 608, 610, 65, 65, 65, 65, + /* 130 */ 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + /* 140 */ 65, 65, 65, 65, 65, 559, 138, 278, 127, 24, + /* 150 */ 24, 24, 24, 24, 24, -72, -72, -72, 228, 341, + /* 160 */ 341, 207, 276, 300, 352, 354, 350, 65, 65, 65, + /* 170 */ 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + /* 180 */ 65, 65, 65, 65, 495, 495, 495, 65, 65, 499, + /* 190 */ 65, 65, 65, 574, 65, 65, 517, 65, 65, 65, + /* 200 */ 65, 65, 65, 65, 65, 65, 65, 566, 691, 288, + /* 210 */ 288, 288, 701, 620, 1058, 675, 603, 964, 964, 967, + /* 220 */ 603, 967, 722, 965, 936, 999, 964, 264, 999, 999, + /* 230 */ 911, 921, 434, 1196, 1119, 1119, 1202, 1202, 1119, 1222, + /* 240 */ 1184, 1126, 1244, 1244, 1244, 1244, 1119, 1251, 1126, 1222, + /* 250 */ 1184, 1184, 1126, 1119, 1251, 1145, 1237, 1119, 1119, 1251, + /* 260 */ 1275, 1119, 1251, 1119, 1251, 1275, 1203, 1203, 1203, 1264, + /* 270 */ 1275, 1203, 1211, 1203, 1264, 1203, 1203, 1194, 1216, 1194, + /* 280 */ 1216, 1194, 1216, 1194, 1216, 1119, 1119, 1206, 1275, 1250, + /* 290 */ 1250, 1275, 1221, 1225, 1224, 1226, 1126, 1336, 1347, 1363, + /* 300 */ 1363, 1371, 1371, 1371, 1371, -72, -72, -72, -72, -72, + /* 310 */ -72, 477, 623, 742, 819, 624, 694, 74, 1023, 221, + /* 320 */ 1055, 1065, 1077, 1087, 1080, 889, 1031, 939, 1093, 1122, + /* 330 */ 1085, 1139, 961, 1024, 1172, 1103, 821, 1384, 1392, 1374, + /* 340 */ 1255, 1385, 1331, 1386, 1378, 1381, 1285, 1277, 1298, 1287, + /* 350 */ 1393, 1289, 1396, 1414, 1293, 1286, 1340, 1341, 1312, 1397, + /* 360 */ 1388, 1304, 1426, 1423, 1407, 1323, 1291, 1379, 1406, 1380, + /* 370 */ 1375, 1391, 1330, 1415, 1418, 1421, 1329, 1337, 1422, 1390, + /* 380 */ 1424, 1425, 1427, 1429, 1395, 1419, 1430, 1400, 1405, 1431, + /* 390 */ 1432, 1433, 1342, 1435, 1436, 1438, 1437, 1339, 1439, 1441, + /* 400 */ 1440, 1434, 1443, 1343, 1445, 1442, 1446, 1444, 1445, 1450, + /* 410 */ 1451, 1452, 1453, 1454, 1459, 1455, 1460, 1462, 1458, 1461, + /* 420 */ 1463, 1466, 1465, 1461, 1467, 1469, 1470, 1471, 1473, 1354, + /* 430 */ 1370, 1376, 1377, 1474, 1485, 1484, }; -#define YY_REDUCE_USE_DFLT (-110) -#define YY_REDUCE_COUNT (305) -#define YY_REDUCE_MIN (-109) -#define YY_REDUCE_MAX (1323) +#define YY_REDUCE_USE_DFLT (-176) +#define YY_REDUCE_COUNT (310) +#define YY_REDUCE_MIN (-175) +#define YY_REDUCE_MAX (1234) static const short yy_reduce_ofst[] = { - /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, - /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, - /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, - /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, - /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, - /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, - /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, - /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, - /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, - /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, - /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, - /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, - /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, - /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, - /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, - /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, - /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, - /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, - /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, - /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, - /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, - /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, - /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, - /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, - /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, - /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, - /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, - /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, - /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, + /* 0 */ -143, 954, 86, 21, -50, 23, 79, 134, 170, -175, + /* 10 */ 229, 260, -121, 212, 219, 291, -54, 349, 362, 156, + /* 20 */ 309, 311, 334, 85, 224, 394, 314, 314, 314, 314, + /* 30 */ 314, 314, 314, 314, 314, 314, 314, 314, 314, 314, + /* 40 */ 314, 314, 314, 314, 314, 314, 314, 314, 314, 314, + /* 50 */ 314, 314, 314, 314, 314, 374, 441, 443, 450, 452, + /* 60 */ 515, 554, 567, 569, 572, 578, 580, 582, 584, 587, + /* 70 */ 593, 631, 644, 646, 649, 655, 657, 659, 661, 664, + /* 80 */ 708, 720, 759, 771, 810, 822, 861, 873, 912, 930, + /* 90 */ 947, 950, 957, 959, 963, 966, 968, 998, 314, 314, + /* 100 */ 314, 314, 314, 314, 314, 314, 314, 447, -53, 166, + /* 110 */ 438, 348, 363, 314, 473, 469, 314, 314, 314, 314, + /* 120 */ -15, 59, 101, 688, 220, 220, 525, 256, 729, 735, + /* 130 */ 736, 740, 741, 744, 645, 448, 738, 458, 786, 503, + /* 140 */ 780, 656, 721, 724, 792, 545, 568, 706, 683, 681, + /* 150 */ 779, 784, 830, 831, 835, 678, 601, -104, -2, 96, + /* 160 */ 111, 218, 287, 308, 310, 312, 335, 411, 453, 461, + /* 170 */ 573, 599, 617, 658, 665, 670, 732, 734, 775, 848, + /* 180 */ 875, 892, 893, 898, 332, 420, 869, 931, 944, 886, + /* 190 */ 983, 992, 1009, 958, 1017, 1028, 988, 1033, 1034, 1035, + /* 200 */ 287, 1036, 1044, 1045, 1047, 1049, 1056, 915, 972, 997, + /* 210 */ 1000, 1002, 886, 1011, 1015, 1061, 1013, 1001, 1003, 977, + /* 220 */ 1018, 979, 1050, 1041, 1040, 1052, 1014, 1004, 1059, 1060, + /* 230 */ 1032, 1038, 1084, 995, 1089, 1090, 1008, 1016, 1092, 1037, + /* 240 */ 1068, 1062, 1069, 1072, 1073, 1074, 1105, 1112, 1071, 1048, + /* 250 */ 1081, 1088, 1078, 1116, 1118, 1046, 1066, 1128, 1136, 1140, + /* 260 */ 1120, 1147, 1146, 1148, 1150, 1130, 1135, 1137, 1138, 1132, + /* 270 */ 1141, 1142, 1143, 1149, 1144, 1153, 1154, 1104, 1107, 1108, + /* 280 */ 1114, 1115, 1117, 1123, 1125, 1173, 1176, 1121, 1165, 1127, + /* 290 */ 1131, 1167, 1157, 1151, 1158, 1166, 1168, 1212, 1214, 1227, + /* 300 */ 1228, 1231, 1232, 1233, 1234, 1152, 1155, 1159, 1198, 1199, + /* 310 */ 1219, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, - /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, - /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, - /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, - /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, - /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, - /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, - /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, - /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, - /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, - /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, - /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, - /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, - /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, - /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, - /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, - /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, - /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, - /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, - /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, - /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, - /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, - /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, - /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, - /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, - /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, - /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, - /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, - /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, - /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, - /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, - /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, - /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, - /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, - /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, - /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, - /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, - /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, - /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, - /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, - /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, - /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, - /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, - /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, - /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, - /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, - /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, - /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, - /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, - /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, - /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, - /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, - /* 640 */ 645, 643, + /* 0 */ 982, 1300, 1300, 1300, 1214, 1214, 1214, 1305, 1300, 1109, + /* 10 */ 1138, 1138, 1274, 1305, 1305, 1305, 1305, 1305, 1305, 1212, + /* 20 */ 1305, 1305, 1305, 1300, 1305, 1113, 1144, 1305, 1305, 1305, + /* 30 */ 1305, 1305, 1305, 1305, 1305, 1273, 1275, 1152, 1151, 1254, + /* 40 */ 1125, 1149, 1142, 1146, 1215, 1208, 1209, 1207, 1211, 1216, + /* 50 */ 1305, 1145, 1177, 1192, 1176, 1305, 1305, 1305, 1305, 1305, + /* 60 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 70 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 80 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 90 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1186, 1191, + /* 100 */ 1198, 1190, 1187, 1179, 1178, 1180, 1181, 1305, 1305, 1008, + /* 110 */ 1074, 1305, 1305, 1182, 1305, 1020, 1183, 1195, 1194, 1193, + /* 120 */ 1015, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 130 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 140 */ 1305, 1305, 1305, 1305, 1305, 982, 1300, 1305, 1305, 1300, + /* 150 */ 1300, 1300, 1300, 1300, 1300, 1292, 1113, 1103, 1305, 1305, + /* 160 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1280, 1278, + /* 170 */ 1305, 1227, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 180 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 190 */ 1305, 1305, 1305, 1109, 1305, 1305, 1305, 1305, 1305, 1305, + /* 200 */ 1305, 1305, 1305, 1305, 1305, 1305, 988, 1305, 1247, 1109, + /* 210 */ 1109, 1109, 1111, 1089, 1101, 990, 1148, 1127, 1127, 1259, + /* 220 */ 1148, 1259, 1045, 1068, 1042, 1138, 1127, 1210, 1138, 1138, + /* 230 */ 1110, 1101, 1305, 1285, 1118, 1118, 1277, 1277, 1118, 1157, + /* 240 */ 1078, 1148, 1085, 1085, 1085, 1085, 1118, 1005, 1148, 1157, + /* 250 */ 1078, 1078, 1148, 1118, 1005, 1253, 1251, 1118, 1118, 1005, + /* 260 */ 1220, 1118, 1005, 1118, 1005, 1220, 1076, 1076, 1076, 1060, + /* 270 */ 1220, 1076, 1045, 1076, 1060, 1076, 1076, 1131, 1126, 1131, + /* 280 */ 1126, 1131, 1126, 1131, 1126, 1118, 1118, 1305, 1220, 1224, + /* 290 */ 1224, 1220, 1143, 1132, 1141, 1139, 1148, 1011, 1063, 998, + /* 300 */ 998, 987, 987, 987, 987, 1297, 1297, 1292, 1047, 1047, + /* 310 */ 1030, 1305, 1305, 1305, 1305, 1305, 1305, 1022, 1305, 1229, + /* 320 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 330 */ 1305, 1305, 1305, 1305, 1305, 1305, 1164, 1305, 983, 1287, + /* 340 */ 1305, 1305, 1284, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 350 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 360 */ 1305, 1257, 1305, 1305, 1305, 1305, 1305, 1305, 1250, 1249, + /* 370 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 380 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 390 */ 1305, 1305, 1092, 1305, 1305, 1305, 1096, 1305, 1305, 1305, + /* 400 */ 1305, 1305, 1305, 1305, 1140, 1305, 1133, 1305, 1213, 1305, + /* 410 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1302, + /* 420 */ 1305, 1305, 1305, 1301, 1305, 1305, 1305, 1305, 1305, 1166, + /* 430 */ 1305, 1165, 1169, 1305, 996, 1305, }; /* The next table maps tokens into fallback tokens. If a construct @@ -121059,9 +126866,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -121167,18 +126978,18 @@ static const char *const yyTokenName[] = { "column", "columnid", "type", "carglist", "typetoken", "typename", "signed", "plus_num", "minus_num", "ccons", "term", "expr", - "onconf", "sortorder", "autoinc", "idxlist_opt", + "onconf", "sortorder", "autoinc", "eidlist_opt", "refargs", "defer_subclause", "refarg", "refact", "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", - "idxlist", "defer_subclause_opt", "orconf", "resolvetype", - "raisetype", "ifexists", "fullname", "selectnowith", - "oneselect", "with", "multiselect_op", "distinct", - "selcollist", "from", "where_opt", "groupby_opt", - "having_opt", "orderby_opt", "limit_opt", "values", - "nexprlist", "exprlist", "sclp", "as", - "seltablist", "stl_prefix", "joinop", "indexed_opt", - "on_opt", "using_opt", "joinop2", "idlist", - "sortlist", "setlist", "insert_cmd", "inscollist_opt", + "sortlist", "eidlist", "defer_subclause_opt", "orconf", + "resolvetype", "raisetype", "ifexists", "fullname", + "selectnowith", "oneselect", "with", "multiselect_op", + "distinct", "selcollist", "from", "where_opt", + "groupby_opt", "having_opt", "orderby_opt", "limit_opt", + "values", "nexprlist", "exprlist", "sclp", + "as", "seltablist", "stl_prefix", "joinop", + "indexed_opt", "on_opt", "using_opt", "joinop2", + "idlist", "setlist", "insert_cmd", "idlist_opt", "likeop", "between_op", "in_op", "case_operand", "case_exprlist", "case_else", "uniqueflag", "collate", "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", @@ -121259,7 +127070,7 @@ static const char *const yyRuleName[] = { /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", /* 63 */ "ccons ::= UNIQUE onconf", /* 64 */ "ccons ::= CHECK LP expr RP", - /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 65 */ "ccons ::= REFERENCES nm eidlist_opt refargs", /* 66 */ "ccons ::= defer_subclause", /* 67 */ "ccons ::= COLLATE ID|STRING", /* 68 */ "autoinc ::=", @@ -121287,10 +127098,10 @@ static const char *const yyRuleName[] = { /* 90 */ "tconscomma ::= COMMA", /* 91 */ "tconscomma ::=", /* 92 */ "tcons ::= CONSTRAINT nm", - /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 93 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 94 */ "tcons ::= UNIQUE LP sortlist RP onconf", /* 95 */ "tcons ::= CHECK LP expr RP onconf", - /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 96 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", /* 97 */ "defer_subclause_opt ::=", /* 98 */ "defer_subclause_opt ::= defer_subclause", /* 99 */ "onconf ::=", @@ -121303,7 +127114,7 @@ static const char *const yyRuleName[] = { /* 106 */ "cmd ::= DROP TABLE ifexists fullname", /* 107 */ "ifexists ::= IF EXISTS", /* 108 */ "ifexists ::=", - /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", + /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", /* 110 */ "cmd ::= DROP VIEW ifexists fullname", /* 111 */ "cmd ::= select", /* 112 */ "select ::= with selectnowith", @@ -121332,195 +127143,196 @@ static const char *const yyRuleName[] = { /* 135 */ "stl_prefix ::= seltablist joinop", /* 136 */ "stl_prefix ::=", /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 140 */ "dbnm ::=", - /* 141 */ "dbnm ::= DOT nm", - /* 142 */ "fullname ::= nm dbnm", - /* 143 */ "joinop ::= COMMA|JOIN", - /* 144 */ "joinop ::= JOIN_KW JOIN", - /* 145 */ "joinop ::= JOIN_KW nm JOIN", - /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 147 */ "on_opt ::= ON expr", - /* 148 */ "on_opt ::=", - /* 149 */ "indexed_opt ::=", - /* 150 */ "indexed_opt ::= INDEXED BY nm", - /* 151 */ "indexed_opt ::= NOT INDEXED", - /* 152 */ "using_opt ::= USING LP idlist RP", - /* 153 */ "using_opt ::=", - /* 154 */ "orderby_opt ::=", - /* 155 */ "orderby_opt ::= ORDER BY sortlist", - /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 157 */ "sortlist ::= expr sortorder", - /* 158 */ "sortorder ::= ASC", - /* 159 */ "sortorder ::= DESC", - /* 160 */ "sortorder ::=", - /* 161 */ "groupby_opt ::=", - /* 162 */ "groupby_opt ::= GROUP BY nexprlist", - /* 163 */ "having_opt ::=", - /* 164 */ "having_opt ::= HAVING expr", - /* 165 */ "limit_opt ::=", - /* 166 */ "limit_opt ::= LIMIT expr", - /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 170 */ "where_opt ::=", - /* 171 */ "where_opt ::= WHERE expr", - /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 173 */ "setlist ::= setlist COMMA nm EQ expr", - /* 174 */ "setlist ::= nm EQ expr", - /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", - /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 177 */ "insert_cmd ::= INSERT orconf", - /* 178 */ "insert_cmd ::= REPLACE", - /* 179 */ "inscollist_opt ::=", - /* 180 */ "inscollist_opt ::= LP idlist RP", - /* 181 */ "idlist ::= idlist COMMA nm", - /* 182 */ "idlist ::= nm", - /* 183 */ "expr ::= term", - /* 184 */ "expr ::= LP expr RP", - /* 185 */ "term ::= NULL", - /* 186 */ "expr ::= ID|INDEXED", - /* 187 */ "expr ::= JOIN_KW", - /* 188 */ "expr ::= nm DOT nm", - /* 189 */ "expr ::= nm DOT nm DOT nm", - /* 190 */ "term ::= INTEGER|FLOAT|BLOB", - /* 191 */ "term ::= STRING", - /* 192 */ "expr ::= VARIABLE", - /* 193 */ "expr ::= expr COLLATE ID|STRING", - /* 194 */ "expr ::= CAST LP expr AS typetoken RP", - /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 196 */ "expr ::= ID|INDEXED LP STAR RP", - /* 197 */ "term ::= CTIME_KW", - /* 198 */ "expr ::= expr AND expr", - /* 199 */ "expr ::= expr OR expr", - /* 200 */ "expr ::= expr LT|GT|GE|LE expr", - /* 201 */ "expr ::= expr EQ|NE expr", - /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 203 */ "expr ::= expr PLUS|MINUS expr", - /* 204 */ "expr ::= expr STAR|SLASH|REM expr", - /* 205 */ "expr ::= expr CONCAT expr", - /* 206 */ "likeop ::= LIKE_KW|MATCH", - /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 208 */ "expr ::= expr likeop expr", - /* 209 */ "expr ::= expr likeop expr ESCAPE expr", - /* 210 */ "expr ::= expr ISNULL|NOTNULL", - /* 211 */ "expr ::= expr NOT NULL", - /* 212 */ "expr ::= expr IS expr", - /* 213 */ "expr ::= expr IS NOT expr", - /* 214 */ "expr ::= NOT expr", - /* 215 */ "expr ::= BITNOT expr", - /* 216 */ "expr ::= MINUS expr", - /* 217 */ "expr ::= PLUS expr", - /* 218 */ "between_op ::= BETWEEN", - /* 219 */ "between_op ::= NOT BETWEEN", - /* 220 */ "expr ::= expr between_op expr AND expr", - /* 221 */ "in_op ::= IN", - /* 222 */ "in_op ::= NOT IN", - /* 223 */ "expr ::= expr in_op LP exprlist RP", - /* 224 */ "expr ::= LP select RP", - /* 225 */ "expr ::= expr in_op LP select RP", - /* 226 */ "expr ::= expr in_op nm dbnm", - /* 227 */ "expr ::= EXISTS LP select RP", - /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 230 */ "case_exprlist ::= WHEN expr THEN expr", - /* 231 */ "case_else ::= ELSE expr", - /* 232 */ "case_else ::=", - /* 233 */ "case_operand ::= expr", - /* 234 */ "case_operand ::=", - /* 235 */ "exprlist ::= nexprlist", - /* 236 */ "exprlist ::=", - /* 237 */ "nexprlist ::= nexprlist COMMA expr", - /* 238 */ "nexprlist ::= expr", - /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", - /* 240 */ "uniqueflag ::= UNIQUE", - /* 241 */ "uniqueflag ::=", - /* 242 */ "idxlist_opt ::=", - /* 243 */ "idxlist_opt ::= LP idxlist RP", - /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 245 */ "idxlist ::= nm collate sortorder", - /* 246 */ "collate ::=", - /* 247 */ "collate ::= COLLATE ID|STRING", - /* 248 */ "cmd ::= DROP INDEX ifexists fullname", - /* 249 */ "cmd ::= VACUUM", - /* 250 */ "cmd ::= VACUUM nm", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "nmnum ::= plus_num", - /* 257 */ "nmnum ::= nm", - /* 258 */ "nmnum ::= ON", - /* 259 */ "nmnum ::= DELETE", - /* 260 */ "nmnum ::= DEFAULT", - /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 262 */ "plus_num ::= INTEGER|FLOAT", - /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 266 */ "trigger_time ::= BEFORE", - /* 267 */ "trigger_time ::= AFTER", - /* 268 */ "trigger_time ::= INSTEAD OF", - /* 269 */ "trigger_time ::=", - /* 270 */ "trigger_event ::= DELETE|INSERT", - /* 271 */ "trigger_event ::= UPDATE", - /* 272 */ "trigger_event ::= UPDATE OF idlist", - /* 273 */ "foreach_clause ::=", - /* 274 */ "foreach_clause ::= FOR EACH ROW", - /* 275 */ "when_clause ::=", - /* 276 */ "when_clause ::= WHEN expr", - /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 279 */ "trnm ::= nm", - /* 280 */ "trnm ::= nm DOT nm", - /* 281 */ "tridxby ::=", - /* 282 */ "tridxby ::= INDEXED BY nm", - /* 283 */ "tridxby ::= NOT INDEXED", - /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 287 */ "trigger_cmd ::= select", - /* 288 */ "expr ::= RAISE LP IGNORE RP", - /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 290 */ "raisetype ::= ROLLBACK", - /* 291 */ "raisetype ::= ABORT", - /* 292 */ "raisetype ::= FAIL", - /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 295 */ "cmd ::= DETACH database_kw_opt expr", - /* 296 */ "key_opt ::=", - /* 297 */ "key_opt ::= KEY expr", - /* 298 */ "database_kw_opt ::= DATABASE", - /* 299 */ "database_kw_opt ::=", - /* 300 */ "cmd ::= REINDEX", - /* 301 */ "cmd ::= REINDEX nm dbnm", - /* 302 */ "cmd ::= ANALYZE", - /* 303 */ "cmd ::= ANALYZE nm dbnm", - /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 306 */ "add_column_fullname ::= fullname", - /* 307 */ "kwcolumn_opt ::=", - /* 308 */ "kwcolumn_opt ::= COLUMNKW", - /* 309 */ "cmd ::= create_vtab", - /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 312 */ "vtabarglist ::= vtabarg", - /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 314 */ "vtabarg ::=", - /* 315 */ "vtabarg ::= vtabarg vtabargtoken", - /* 316 */ "vtabargtoken ::= ANY", - /* 317 */ "vtabargtoken ::= lp anylist RP", - /* 318 */ "lp ::= LP", - /* 319 */ "anylist ::=", - /* 320 */ "anylist ::= anylist LP anylist RP", - /* 321 */ "anylist ::= anylist ANY", - /* 322 */ "with ::=", - /* 323 */ "with ::= WITH wqlist", - /* 324 */ "with ::= WITH RECURSIVE wqlist", - /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", - /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", + /* 138 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 139 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 140 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 141 */ "dbnm ::=", + /* 142 */ "dbnm ::= DOT nm", + /* 143 */ "fullname ::= nm dbnm", + /* 144 */ "joinop ::= COMMA|JOIN", + /* 145 */ "joinop ::= JOIN_KW JOIN", + /* 146 */ "joinop ::= JOIN_KW nm JOIN", + /* 147 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 148 */ "on_opt ::= ON expr", + /* 149 */ "on_opt ::=", + /* 150 */ "indexed_opt ::=", + /* 151 */ "indexed_opt ::= INDEXED BY nm", + /* 152 */ "indexed_opt ::= NOT INDEXED", + /* 153 */ "using_opt ::= USING LP idlist RP", + /* 154 */ "using_opt ::=", + /* 155 */ "orderby_opt ::=", + /* 156 */ "orderby_opt ::= ORDER BY sortlist", + /* 157 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 158 */ "sortlist ::= expr sortorder", + /* 159 */ "sortorder ::= ASC", + /* 160 */ "sortorder ::= DESC", + /* 161 */ "sortorder ::=", + /* 162 */ "groupby_opt ::=", + /* 163 */ "groupby_opt ::= GROUP BY nexprlist", + /* 164 */ "having_opt ::=", + /* 165 */ "having_opt ::= HAVING expr", + /* 166 */ "limit_opt ::=", + /* 167 */ "limit_opt ::= LIMIT expr", + /* 168 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 169 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 170 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 171 */ "where_opt ::=", + /* 172 */ "where_opt ::= WHERE expr", + /* 173 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 174 */ "setlist ::= setlist COMMA nm EQ expr", + /* 175 */ "setlist ::= nm EQ expr", + /* 176 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", + /* 177 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", + /* 178 */ "insert_cmd ::= INSERT orconf", + /* 179 */ "insert_cmd ::= REPLACE", + /* 180 */ "idlist_opt ::=", + /* 181 */ "idlist_opt ::= LP idlist RP", + /* 182 */ "idlist ::= idlist COMMA nm", + /* 183 */ "idlist ::= nm", + /* 184 */ "expr ::= term", + /* 185 */ "expr ::= LP expr RP", + /* 186 */ "term ::= NULL", + /* 187 */ "expr ::= ID|INDEXED", + /* 188 */ "expr ::= JOIN_KW", + /* 189 */ "expr ::= nm DOT nm", + /* 190 */ "expr ::= nm DOT nm DOT nm", + /* 191 */ "term ::= INTEGER|FLOAT|BLOB", + /* 192 */ "term ::= STRING", + /* 193 */ "expr ::= VARIABLE", + /* 194 */ "expr ::= expr COLLATE ID|STRING", + /* 195 */ "expr ::= CAST LP expr AS typetoken RP", + /* 196 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 197 */ "expr ::= ID|INDEXED LP STAR RP", + /* 198 */ "term ::= CTIME_KW", + /* 199 */ "expr ::= expr AND expr", + /* 200 */ "expr ::= expr OR expr", + /* 201 */ "expr ::= expr LT|GT|GE|LE expr", + /* 202 */ "expr ::= expr EQ|NE expr", + /* 203 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 204 */ "expr ::= expr PLUS|MINUS expr", + /* 205 */ "expr ::= expr STAR|SLASH|REM expr", + /* 206 */ "expr ::= expr CONCAT expr", + /* 207 */ "likeop ::= LIKE_KW|MATCH", + /* 208 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 209 */ "expr ::= expr likeop expr", + /* 210 */ "expr ::= expr likeop expr ESCAPE expr", + /* 211 */ "expr ::= expr ISNULL|NOTNULL", + /* 212 */ "expr ::= expr NOT NULL", + /* 213 */ "expr ::= expr IS expr", + /* 214 */ "expr ::= expr IS NOT expr", + /* 215 */ "expr ::= NOT expr", + /* 216 */ "expr ::= BITNOT expr", + /* 217 */ "expr ::= MINUS expr", + /* 218 */ "expr ::= PLUS expr", + /* 219 */ "between_op ::= BETWEEN", + /* 220 */ "between_op ::= NOT BETWEEN", + /* 221 */ "expr ::= expr between_op expr AND expr", + /* 222 */ "in_op ::= IN", + /* 223 */ "in_op ::= NOT IN", + /* 224 */ "expr ::= expr in_op LP exprlist RP", + /* 225 */ "expr ::= LP select RP", + /* 226 */ "expr ::= expr in_op LP select RP", + /* 227 */ "expr ::= expr in_op nm dbnm", + /* 228 */ "expr ::= EXISTS LP select RP", + /* 229 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 230 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 231 */ "case_exprlist ::= WHEN expr THEN expr", + /* 232 */ "case_else ::= ELSE expr", + /* 233 */ "case_else ::=", + /* 234 */ "case_operand ::= expr", + /* 235 */ "case_operand ::=", + /* 236 */ "exprlist ::= nexprlist", + /* 237 */ "exprlist ::=", + /* 238 */ "nexprlist ::= nexprlist COMMA expr", + /* 239 */ "nexprlist ::= expr", + /* 240 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 241 */ "uniqueflag ::= UNIQUE", + /* 242 */ "uniqueflag ::=", + /* 243 */ "eidlist_opt ::=", + /* 244 */ "eidlist_opt ::= LP eidlist RP", + /* 245 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 246 */ "eidlist ::= nm collate sortorder", + /* 247 */ "collate ::=", + /* 248 */ "collate ::= COLLATE ID|STRING", + /* 249 */ "cmd ::= DROP INDEX ifexists fullname", + /* 250 */ "cmd ::= VACUUM", + /* 251 */ "cmd ::= VACUUM nm", + /* 252 */ "cmd ::= PRAGMA nm dbnm", + /* 253 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 254 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 255 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 256 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 257 */ "nmnum ::= plus_num", + /* 258 */ "nmnum ::= nm", + /* 259 */ "nmnum ::= ON", + /* 260 */ "nmnum ::= DELETE", + /* 261 */ "nmnum ::= DEFAULT", + /* 262 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 263 */ "plus_num ::= INTEGER|FLOAT", + /* 264 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 265 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 266 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 267 */ "trigger_time ::= BEFORE", + /* 268 */ "trigger_time ::= AFTER", + /* 269 */ "trigger_time ::= INSTEAD OF", + /* 270 */ "trigger_time ::=", + /* 271 */ "trigger_event ::= DELETE|INSERT", + /* 272 */ "trigger_event ::= UPDATE", + /* 273 */ "trigger_event ::= UPDATE OF idlist", + /* 274 */ "foreach_clause ::=", + /* 275 */ "foreach_clause ::= FOR EACH ROW", + /* 276 */ "when_clause ::=", + /* 277 */ "when_clause ::= WHEN expr", + /* 278 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 279 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 280 */ "trnm ::= nm", + /* 281 */ "trnm ::= nm DOT nm", + /* 282 */ "tridxby ::=", + /* 283 */ "tridxby ::= INDEXED BY nm", + /* 284 */ "tridxby ::= NOT INDEXED", + /* 285 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 286 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", + /* 287 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 288 */ "trigger_cmd ::= select", + /* 289 */ "expr ::= RAISE LP IGNORE RP", + /* 290 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 291 */ "raisetype ::= ROLLBACK", + /* 292 */ "raisetype ::= ABORT", + /* 293 */ "raisetype ::= FAIL", + /* 294 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 295 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 296 */ "cmd ::= DETACH database_kw_opt expr", + /* 297 */ "key_opt ::=", + /* 298 */ "key_opt ::= KEY expr", + /* 299 */ "database_kw_opt ::= DATABASE", + /* 300 */ "database_kw_opt ::=", + /* 301 */ "cmd ::= REINDEX", + /* 302 */ "cmd ::= REINDEX nm dbnm", + /* 303 */ "cmd ::= ANALYZE", + /* 304 */ "cmd ::= ANALYZE nm dbnm", + /* 305 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 306 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 307 */ "add_column_fullname ::= fullname", + /* 308 */ "kwcolumn_opt ::=", + /* 309 */ "kwcolumn_opt ::= COLUMNKW", + /* 310 */ "cmd ::= create_vtab", + /* 311 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 312 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 313 */ "vtabarglist ::= vtabarg", + /* 314 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 315 */ "vtabarg ::=", + /* 316 */ "vtabarg ::= vtabarg vtabargtoken", + /* 317 */ "vtabargtoken ::= ANY", + /* 318 */ "vtabargtoken ::= lp anylist RP", + /* 319 */ "lp ::= LP", + /* 320 */ "anylist ::=", + /* 321 */ "anylist ::= anylist LP anylist RP", + /* 322 */ "anylist ::= anylist ANY", + /* 323 */ "with ::=", + /* 324 */ "with ::= WITH wqlist", + /* 325 */ "with ::= WITH RECURSIVE wqlist", + /* 326 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 327 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", }; #endif /* NDEBUG */ @@ -121600,9 +127412,9 @@ static void yy_destructor( ** inside the C code. */ case 163: /* select */ - case 195: /* selectnowith */ - case 196: /* oneselect */ - case 207: /* values */ + case 196: /* selectnowith */ + case 197: /* oneselect */ + case 208: /* values */ { sqlite3SelectDelete(pParse->db, (yypminor->yy3)); } @@ -121613,38 +127425,38 @@ sqlite3SelectDelete(pParse->db, (yypminor->yy3)); sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); } break; - case 179: /* idxlist_opt */ - case 188: /* idxlist */ - case 200: /* selcollist */ - case 203: /* groupby_opt */ - case 205: /* orderby_opt */ - case 208: /* nexprlist */ - case 209: /* exprlist */ - case 210: /* sclp */ - case 220: /* sortlist */ + case 179: /* eidlist_opt */ + case 188: /* sortlist */ + case 189: /* eidlist */ + case 201: /* selcollist */ + case 204: /* groupby_opt */ + case 206: /* orderby_opt */ + case 209: /* nexprlist */ + case 210: /* exprlist */ + case 211: /* sclp */ case 221: /* setlist */ case 228: /* case_exprlist */ { sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 194: /* fullname */ - case 201: /* from */ - case 212: /* seltablist */ - case 213: /* stl_prefix */ + case 195: /* fullname */ + case 202: /* from */ + case 213: /* seltablist */ + case 214: /* stl_prefix */ { sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); } break; - case 197: /* with */ + case 198: /* with */ case 252: /* wqlist */ { sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 202: /* where_opt */ - case 204: /* having_opt */ - case 216: /* on_opt */ + case 203: /* where_opt */ + case 205: /* having_opt */ + case 217: /* on_opt */ case 227: /* case_operand */ case 229: /* case_else */ case 238: /* when_clause */ @@ -121653,9 +127465,9 @@ sqlite3WithDelete(pParse->db, (yypminor->yy59)); sqlite3ExprDelete(pParse->db, (yypminor->yy132)); } break; - case 217: /* using_opt */ - case 219: /* idlist */ - case 223: /* inscollist_opt */ + case 218: /* using_opt */ + case 220: /* idlist */ + case 223: /* idlist_opt */ { sqlite3IdListDelete(pParse->db, (yypminor->yy408)); } @@ -121689,7 +127501,7 @@ static int yy_pop_parser_stack(yyParser *pParser){ /* There is no mechanism by which the parser stack can be popped below ** empty in SQLite. */ - if( NEVER(pParser->yyidx<0) ) return 0; + assert( pParser->yyidx>=0 ); #ifndef NDEBUG if( yyTraceFILE && pParser->yyidx>=0 ){ fprintf(yyTraceFILE,"%sPopping %s\n", @@ -121755,10 +127567,10 @@ static int yy_find_shift_action( int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + if( stateno>=YY_MIN_REDUCE ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ @@ -121861,7 +127673,29 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + +/* +** Perform a shift action. Return the number of errors. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ @@ -121894,16 +127728,7 @@ static void yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - } -#endif + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -122010,81 +127835,82 @@ static const struct { { 187, 5 }, { 187, 5 }, { 187, 10 }, - { 189, 0 }, - { 189, 1 }, + { 190, 0 }, + { 190, 1 }, { 176, 0 }, { 176, 3 }, - { 190, 0 }, - { 190, 2 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, + { 191, 0 }, + { 191, 2 }, + { 192, 1 }, + { 192, 1 }, + { 192, 1 }, { 149, 4 }, - { 193, 2 }, - { 193, 0 }, - { 149, 8 }, + { 194, 2 }, + { 194, 0 }, + { 149, 9 }, { 149, 4 }, { 149, 1 }, { 163, 2 }, - { 195, 1 }, - { 195, 3 }, - { 198, 1 }, - { 198, 2 }, - { 198, 1 }, - { 196, 9 }, { 196, 1 }, - { 207, 4 }, - { 207, 5 }, + { 196, 3 }, { 199, 1 }, + { 199, 2 }, { 199, 1 }, - { 199, 0 }, - { 210, 2 }, - { 210, 0 }, - { 200, 3 }, - { 200, 2 }, - { 200, 4 }, + { 197, 9 }, + { 197, 1 }, + { 208, 4 }, + { 208, 5 }, + { 200, 1 }, + { 200, 1 }, + { 200, 0 }, { 211, 2 }, - { 211, 1 }, { 211, 0 }, - { 201, 0 }, + { 201, 3 }, { 201, 2 }, - { 213, 2 }, - { 213, 0 }, - { 212, 7 }, - { 212, 7 }, - { 212, 7 }, + { 201, 4 }, + { 212, 2 }, + { 212, 1 }, + { 212, 0 }, + { 202, 0 }, + { 202, 2 }, + { 214, 2 }, + { 214, 0 }, + { 213, 7 }, + { 213, 9 }, + { 213, 7 }, + { 213, 7 }, { 159, 0 }, { 159, 2 }, - { 194, 2 }, - { 214, 1 }, - { 214, 2 }, - { 214, 3 }, - { 214, 4 }, - { 216, 2 }, - { 216, 0 }, - { 215, 0 }, - { 215, 3 }, + { 195, 2 }, + { 215, 1 }, { 215, 2 }, - { 217, 4 }, + { 215, 3 }, + { 215, 4 }, + { 217, 2 }, { 217, 0 }, - { 205, 0 }, - { 205, 3 }, - { 220, 4 }, - { 220, 2 }, + { 216, 0 }, + { 216, 3 }, + { 216, 2 }, + { 218, 4 }, + { 218, 0 }, + { 206, 0 }, + { 206, 3 }, + { 188, 4 }, + { 188, 2 }, { 177, 1 }, { 177, 1 }, { 177, 0 }, - { 203, 0 }, - { 203, 3 }, { 204, 0 }, - { 204, 2 }, - { 206, 0 }, - { 206, 2 }, - { 206, 4 }, - { 206, 4 }, + { 204, 3 }, + { 205, 0 }, + { 205, 2 }, + { 207, 0 }, + { 207, 2 }, + { 207, 4 }, + { 207, 4 }, { 149, 6 }, - { 202, 0 }, - { 202, 2 }, + { 203, 0 }, + { 203, 2 }, { 149, 8 }, { 221, 5 }, { 221, 3 }, @@ -122094,8 +127920,8 @@ static const struct { { 222, 1 }, { 223, 0 }, { 223, 3 }, - { 219, 3 }, - { 219, 1 }, + { 220, 3 }, + { 220, 1 }, { 175, 1 }, { 175, 3 }, { 174, 1 }, @@ -122148,17 +127974,17 @@ static const struct { { 229, 0 }, { 227, 1 }, { 227, 0 }, + { 210, 1 }, + { 210, 0 }, + { 209, 3 }, { 209, 1 }, - { 209, 0 }, - { 208, 3 }, - { 208, 1 }, { 149, 12 }, { 230, 1 }, { 230, 0 }, { 179, 0 }, { 179, 3 }, - { 188, 5 }, - { 188, 3 }, + { 189, 5 }, + { 189, 3 }, { 231, 0 }, { 231, 2 }, { 149, 4 }, @@ -122203,9 +128029,9 @@ static const struct { { 239, 1 }, { 175, 4 }, { 175, 6 }, - { 192, 1 }, - { 192, 1 }, - { 192, 1 }, + { 193, 1 }, + { 193, 1 }, + { 193, 1 }, { 149, 4 }, { 149, 6 }, { 149, 3 }, @@ -122235,9 +128061,9 @@ static const struct { { 251, 0 }, { 251, 4 }, { 251, 2 }, - { 197, 0 }, - { 197, 2 }, - { 197, 3 }, + { 198, 0 }, + { 198, 2 }, + { 198, 3 }, { 252, 6 }, { 252, 8 }, }; @@ -122262,8 +128088,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -122360,8 +128187,9 @@ static void yy_reduce( case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); case 108: /* ifexists ::= */ yytestcase(yyruleno==108); - case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); - case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); + case 219: /* between_op ::= BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= */ yytestcase(yyruleno==247); {yygotominor.yy328 = 0;} break; case 29: /* ifnotexists ::= IF NOT EXISTS */ @@ -122369,8 +128197,9 @@ static void yy_reduce( case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); - case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); - case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 220: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==220); + case 223: /* in_op ::= NOT IN */ yytestcase(yyruleno==223); + case 248: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==248); {yygotominor.yy328 = 1;} break; case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ @@ -122390,7 +128219,7 @@ static void yy_reduce( case 35: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yygotominor.yy186 = TF_WithoutRowid; + yygotominor.yy186 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ yygotominor.yy186 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); @@ -122417,18 +128246,17 @@ static void yy_reduce( case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); - case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); - case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); - case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); - case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); - case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); - case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); - case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); - case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); - case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); - case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); - case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); - case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); + case 142: /* dbnm ::= DOT nm */ yytestcase(yyruleno==142); + case 151: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==151); + case 257: /* nmnum ::= plus_num */ yytestcase(yyruleno==257); + case 258: /* nmnum ::= nm */ yytestcase(yyruleno==258); + case 259: /* nmnum ::= ON */ yytestcase(yyruleno==259); + case 260: /* nmnum ::= DELETE */ yytestcase(yyruleno==260); + case 261: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==261); + case 262: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==262); + case 263: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==263); + case 264: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==264); + case 280: /* trnm ::= nm */ yytestcase(yyruleno==280); {yygotominor.yy0 = yymsp[0].minor.yy0;} break; case 44: /* type ::= typetoken */ @@ -122488,7 +128316,7 @@ static void yy_reduce( case 64: /* ccons ::= CHECK LP expr RP */ {sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} break; - case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ + case 65: /* ccons ::= REFERENCES nm eidlist_opt refargs */ {sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} break; case 66: /* ccons ::= defer_subclause */ @@ -122543,16 +128371,16 @@ static void yy_reduce( case 90: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ + case 93: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ {sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} break; - case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ + case 94: /* tcons ::= UNIQUE LP sortlist RP onconf */ {sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} break; case 95: /* tcons ::= CHECK LP expr RP onconf */ {sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} break; - case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 96: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); @@ -122578,9 +128406,9 @@ static void yy_reduce( sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); } break; - case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy3, yymsp[-7].minor.yy328, yymsp[-5].minor.yy328); } break; case 110: /* cmd ::= DROP VIEW ifexists fullname */ @@ -122597,21 +128425,10 @@ static void yy_reduce( break; case 112: /* select ::= with selectnowith */ { - Select *p = yymsp[0].minor.yy3, *pNext, *pLoop; + Select *p = yymsp[0].minor.yy3; if( p ){ - int cnt = 0, mxSelect; p->pWith = yymsp[-1].minor.yy59; - if( p->pPrior ){ - pNext = 0; - for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ - pLoop->pNext = pNext; - pLoop->selFlags |= SF_Compound; - } - mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; - if( mxSelect && cnt>mxSelect ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - } - } + parserDoubleLinkSelect(pParse, p); }else{ sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); } @@ -122625,19 +128442,23 @@ static void yy_reduce( case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ { Select *pRhs = yymsp[0].minor.yy3; + Select *pLhs = yymsp[-2].minor.yy3; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); } if( pRhs ){ pRhs->op = (u8)yymsp[-1].minor.yy328; - pRhs->pPrior = yymsp[-2].minor.yy3; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; + pRhs->selFlags &= ~SF_MultiValue; if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); + sqlite3SelectDelete(pParse->db, pLhs); } yygotominor.yy3 = pRhs; } @@ -122681,13 +128502,16 @@ static void yy_reduce( break; case 121: /* values ::= values COMMA LP exprlist RP */ { - Select *pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + Select *pRight, *pLeft = yymsp[-4].minor.yy3; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; - pRight->pPrior = yymsp[-4].minor.yy3; + pLeft = yymsp[-4].minor.yy3; + pRight->pPrior = pLeft; yygotominor.yy3 = pRight; }else{ - yygotominor.yy3 = yymsp[-4].minor.yy3; + yygotominor.yy3 = pLeft; } } break; @@ -122695,18 +128519,20 @@ static void yy_reduce( {yygotominor.yy381 = SF_Distinct;} break; case 123: /* distinct ::= ALL */ - case 124: /* distinct ::= */ yytestcase(yyruleno==124); +{yygotominor.yy381 = SF_All;} + break; + case 124: /* distinct ::= */ {yygotominor.yy381 = 0;} break; case 125: /* sclp ::= selcollist COMMA */ - case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); + case 244: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==244); {yygotominor.yy14 = yymsp[-1].minor.yy14;} break; case 126: /* sclp ::= */ - case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); - case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); - case 236: /* exprlist ::= */ yytestcase(yyruleno==236); - case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); + case 155: /* orderby_opt ::= */ yytestcase(yyruleno==155); + case 162: /* groupby_opt ::= */ yytestcase(yyruleno==162); + case 237: /* exprlist ::= */ yytestcase(yyruleno==237); + case 243: /* eidlist_opt ::= */ yytestcase(yyruleno==243); {yygotominor.yy14 = 0;} break; case 127: /* selcollist ::= sclp expr as */ @@ -122745,7 +128571,7 @@ static void yy_reduce( case 135: /* stl_prefix ::= seltablist joinop */ { yygotominor.yy65 = yymsp[-1].minor.yy65; - if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; + if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy328; } break; case 136: /* stl_prefix ::= */ @@ -122757,12 +128583,18 @@ static void yy_reduce( sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); } break; - case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 138: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ +{ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy65,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListFuncArgs(pParse, yygotominor.yy65, yymsp[-4].minor.yy14); +} + break; + case 139: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); } break; - case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 140: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ yygotominor.yy65 = yymsp[-4].minor.yy65; @@ -122786,94 +128618,96 @@ static void yy_reduce( } } break; - case 140: /* dbnm ::= */ - case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); + case 141: /* dbnm ::= */ + case 150: /* indexed_opt ::= */ yytestcase(yyruleno==150); {yygotominor.yy0.z=0; yygotominor.yy0.n=0;} break; - case 142: /* fullname ::= nm dbnm */ + case 143: /* fullname ::= nm dbnm */ {yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; - case 143: /* joinop ::= COMMA|JOIN */ + case 144: /* joinop ::= COMMA|JOIN */ { yygotominor.yy328 = JT_INNER; } break; - case 144: /* joinop ::= JOIN_KW JOIN */ + case 145: /* joinop ::= JOIN_KW JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } break; - case 145: /* joinop ::= JOIN_KW nm JOIN */ + case 146: /* joinop ::= JOIN_KW nm JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } break; - case 146: /* joinop ::= JOIN_KW nm nm JOIN */ + case 147: /* joinop ::= JOIN_KW nm nm JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } break; - case 147: /* on_opt ::= ON expr */ - case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); - case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); - case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); - case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); + case 148: /* on_opt ::= ON expr */ + case 165: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==165); + case 172: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==172); + case 232: /* case_else ::= ELSE expr */ yytestcase(yyruleno==232); + case 234: /* case_operand ::= expr */ yytestcase(yyruleno==234); {yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} break; - case 148: /* on_opt ::= */ - case 163: /* having_opt ::= */ yytestcase(yyruleno==163); - case 170: /* where_opt ::= */ yytestcase(yyruleno==170); - case 232: /* case_else ::= */ yytestcase(yyruleno==232); - case 234: /* case_operand ::= */ yytestcase(yyruleno==234); + case 149: /* on_opt ::= */ + case 164: /* having_opt ::= */ yytestcase(yyruleno==164); + case 171: /* where_opt ::= */ yytestcase(yyruleno==171); + case 233: /* case_else ::= */ yytestcase(yyruleno==233); + case 235: /* case_operand ::= */ yytestcase(yyruleno==235); {yygotominor.yy132 = 0;} break; - case 151: /* indexed_opt ::= NOT INDEXED */ + case 152: /* indexed_opt ::= NOT INDEXED */ {yygotominor.yy0.z=0; yygotominor.yy0.n=1;} break; - case 152: /* using_opt ::= USING LP idlist RP */ - case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); + case 153: /* using_opt ::= USING LP idlist RP */ + case 181: /* idlist_opt ::= LP idlist RP */ yytestcase(yyruleno==181); {yygotominor.yy408 = yymsp[-1].minor.yy408;} break; - case 153: /* using_opt ::= */ - case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); + case 154: /* using_opt ::= */ + case 180: /* idlist_opt ::= */ yytestcase(yyruleno==180); {yygotominor.yy408 = 0;} break; - case 155: /* orderby_opt ::= ORDER BY sortlist */ - case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); - case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); + case 156: /* orderby_opt ::= ORDER BY sortlist */ + case 163: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==163); + case 236: /* exprlist ::= nexprlist */ yytestcase(yyruleno==236); {yygotominor.yy14 = yymsp[0].minor.yy14;} break; - case 156: /* sortlist ::= sortlist COMMA expr sortorder */ + case 157: /* sortlist ::= sortlist COMMA expr sortorder */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328); } break; - case 157: /* sortlist ::= expr sortorder */ + case 158: /* sortlist ::= expr sortorder */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; + sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328); } break; - case 158: /* sortorder ::= ASC */ - case 160: /* sortorder ::= */ yytestcase(yyruleno==160); + case 159: /* sortorder ::= ASC */ {yygotominor.yy328 = SQLITE_SO_ASC;} break; - case 159: /* sortorder ::= DESC */ + case 160: /* sortorder ::= DESC */ {yygotominor.yy328 = SQLITE_SO_DESC;} break; - case 165: /* limit_opt ::= */ + case 161: /* sortorder ::= */ +{yygotominor.yy328 = SQLITE_SO_UNDEFINED;} + break; + case 166: /* limit_opt ::= */ {yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} break; - case 166: /* limit_opt ::= LIMIT expr */ + case 167: /* limit_opt ::= LIMIT expr */ {yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} break; - case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ + case 168: /* limit_opt ::= LIMIT expr OFFSET expr */ {yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} break; - case 168: /* limit_opt ::= LIMIT expr COMMA expr */ + case 169: /* limit_opt ::= LIMIT expr COMMA expr */ {yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} break; - case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ + case 170: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ { sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); } break; - case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 173: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ { sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); @@ -122881,58 +128715,58 @@ static void yy_reduce( sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); } break; - case 173: /* setlist ::= setlist COMMA nm EQ expr */ + case 174: /* setlist ::= setlist COMMA nm EQ expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 174: /* setlist ::= nm EQ expr */ + case 175: /* setlist ::= nm EQ expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ + case 176: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ { sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); } break; - case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ + case 177: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ { sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); } break; - case 177: /* insert_cmd ::= INSERT orconf */ + case 178: /* insert_cmd ::= INSERT orconf */ {yygotominor.yy186 = yymsp[0].minor.yy186;} break; - case 178: /* insert_cmd ::= REPLACE */ + case 179: /* insert_cmd ::= REPLACE */ {yygotominor.yy186 = OE_Replace;} break; - case 181: /* idlist ::= idlist COMMA nm */ + case 182: /* idlist ::= idlist COMMA nm */ {yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} break; - case 182: /* idlist ::= nm */ + case 183: /* idlist ::= nm */ {yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} break; - case 183: /* expr ::= term */ + case 184: /* expr ::= term */ {yygotominor.yy346 = yymsp[0].minor.yy346;} break; - case 184: /* expr ::= LP expr RP */ + case 185: /* expr ::= LP expr RP */ {yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} break; - case 185: /* term ::= NULL */ - case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); - case 191: /* term ::= STRING */ yytestcase(yyruleno==191); + case 186: /* term ::= NULL */ + case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); + case 192: /* term ::= STRING */ yytestcase(yyruleno==192); {spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} break; - case 186: /* expr ::= ID|INDEXED */ - case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); + case 187: /* expr ::= ID|INDEXED */ + case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); {spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} break; - case 188: /* expr ::= nm DOT nm */ + case 189: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); @@ -122940,7 +128774,7 @@ static void yy_reduce( spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } break; - case 189: /* expr ::= nm DOT nm DOT nm */ + case 190: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); @@ -122950,7 +128784,7 @@ static void yy_reduce( spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); } break; - case 192: /* expr ::= VARIABLE */ + case 193: /* expr ::= VARIABLE */ { if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ /* When doing a nested parse, one can include terms in an expression @@ -122970,60 +128804,60 @@ static void yy_reduce( spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 193: /* expr ::= expr COLLATE ID|STRING */ + case 194: /* expr ::= expr COLLATE ID|STRING */ { - yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0, 1); yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 194: /* expr ::= CAST LP expr AS typetoken RP */ + case 195: /* expr ::= CAST LP expr AS typetoken RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); } break; - case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 196: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ + if( yymsp[-2].minor.yy381==SF_Distinct && yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->flags |= EP_Distinct; } } break; - case 196: /* expr ::= ID|INDEXED LP STAR RP */ + case 197: /* expr ::= ID|INDEXED LP STAR RP */ { yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } break; - case 197: /* term ::= CTIME_KW */ + case 198: /* term ::= CTIME_KW */ { yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 198: /* expr ::= expr AND expr */ - case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); + case 199: /* expr ::= expr AND expr */ + case 200: /* expr ::= expr OR expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==205); + case 206: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==206); {spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} break; - case 206: /* likeop ::= LIKE_KW|MATCH */ + case 207: /* likeop ::= LIKE_KW|MATCH */ {yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} break; - case 207: /* likeop ::= NOT LIKE_KW|MATCH */ + case 208: /* likeop ::= NOT LIKE_KW|MATCH */ {yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} break; - case 208: /* expr ::= expr likeop expr */ + case 209: /* expr ::= expr likeop expr */ { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); @@ -123035,7 +128869,7 @@ static void yy_reduce( if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 209: /* expr ::= expr likeop expr ESCAPE expr */ + case 210: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); @@ -123048,35 +128882,35 @@ static void yy_reduce( if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 210: /* expr ::= expr ISNULL|NOTNULL */ + case 211: /* expr ::= expr ISNULL|NOTNULL */ {spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} break; - case 211: /* expr ::= expr NOT NULL */ + case 212: /* expr ::= expr NOT NULL */ {spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} break; - case 212: /* expr ::= expr IS expr */ + case 213: /* expr ::= expr IS expr */ { spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); } break; - case 213: /* expr ::= expr IS NOT expr */ + case 214: /* expr ::= expr IS NOT expr */ { spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); } break; - case 214: /* expr ::= NOT expr */ - case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); + case 215: /* expr ::= NOT expr */ + case 216: /* expr ::= BITNOT expr */ yytestcase(yyruleno==216); {spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 216: /* expr ::= MINUS expr */ + case 217: /* expr ::= MINUS expr */ {spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 217: /* expr ::= PLUS expr */ + case 218: /* expr ::= PLUS expr */ {spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 220: /* expr ::= expr between_op expr AND expr */ + case 221: /* expr ::= expr between_op expr AND expr */ { ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); @@ -123091,7 +128925,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; } break; - case 223: /* expr ::= expr in_op LP exprlist RP */ + case 224: /* expr ::= expr in_op LP exprlist RP */ { if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form @@ -123135,7 +128969,7 @@ static void yy_reduce( yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); if( yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); } @@ -123145,13 +128979,13 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 224: /* expr ::= LP select RP */ + case 225: /* expr ::= LP select RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); if( yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } @@ -123159,13 +128993,13 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 225: /* expr ::= expr in_op LP select RP */ + case 226: /* expr ::= expr in_op LP select RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); if( yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } @@ -123174,14 +129008,14 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 226: /* expr ::= expr in_op nm dbnm */ + case 227: /* expr ::= expr in_op nm dbnm */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); if( yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ sqlite3SrcListDelete(pParse->db, pSrc); } @@ -123190,13 +129024,13 @@ static void yy_reduce( yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; } break; - case 227: /* expr ::= EXISTS LP select RP */ + case 228: /* expr ::= EXISTS LP select RP */ { Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ p->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(p, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, p); + ExprSetProperty(p, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, p); }else{ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } @@ -123204,12 +129038,12 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 229: /* expr ::= CASE case_operand case_exprlist case_else END */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); if( yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; - sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); @@ -123218,82 +129052,71 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 230: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 230: /* case_exprlist ::= WHEN expr THEN expr */ + case 231: /* case_exprlist ::= WHEN expr THEN expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 237: /* nexprlist ::= nexprlist COMMA expr */ + case 238: /* nexprlist ::= nexprlist COMMA expr */ {yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} break; - case 238: /* nexprlist ::= expr */ + case 239: /* nexprlist ::= expr */ {yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} break; - case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ + case 240: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); } break; - case 240: /* uniqueflag ::= UNIQUE */ - case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); + case 241: /* uniqueflag ::= UNIQUE */ + case 292: /* raisetype ::= ABORT */ yytestcase(yyruleno==292); {yygotominor.yy328 = OE_Abort;} break; - case 241: /* uniqueflag ::= */ + case 242: /* uniqueflag ::= */ {yygotominor.yy328 = OE_None;} break; - case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ + case 245: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); - sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yygotominor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328); } break; - case 245: /* idxlist ::= nm collate sortorder */ + case 246: /* eidlist ::= nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yygotominor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328); } break; - case 246: /* collate ::= */ -{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} - break; - case 248: /* cmd ::= DROP INDEX ifexists fullname */ + case 249: /* cmd ::= DROP INDEX ifexists fullname */ {sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} break; - case 249: /* cmd ::= VACUUM */ - case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); + case 250: /* cmd ::= VACUUM */ + case 251: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==251); {sqlite3Vacuum(pParse);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 252: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 253: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 254: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 255: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 256: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 265: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; @@ -123301,38 +129124,38 @@ static void yy_reduce( sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); } break; - case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 266: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); } break; - case 266: /* trigger_time ::= BEFORE */ - case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); + case 267: /* trigger_time ::= BEFORE */ + case 270: /* trigger_time ::= */ yytestcase(yyruleno==270); { yygotominor.yy328 = TK_BEFORE; } break; - case 267: /* trigger_time ::= AFTER */ + case 268: /* trigger_time ::= AFTER */ { yygotominor.yy328 = TK_AFTER; } break; - case 268: /* trigger_time ::= INSTEAD OF */ + case 269: /* trigger_time ::= INSTEAD OF */ { yygotominor.yy328 = TK_INSTEAD;} break; - case 270: /* trigger_event ::= DELETE|INSERT */ - case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); + case 271: /* trigger_event ::= DELETE|INSERT */ + case 272: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==272); {yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} break; - case 272: /* trigger_event ::= UPDATE OF idlist */ + case 273: /* trigger_event ::= UPDATE OF idlist */ {yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} break; - case 275: /* when_clause ::= */ - case 296: /* key_opt ::= */ yytestcase(yyruleno==296); + case 276: /* when_clause ::= */ + case 297: /* key_opt ::= */ yytestcase(yyruleno==297); { yygotominor.yy132 = 0; } break; - case 276: /* when_clause ::= WHEN expr */ - case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); + case 277: /* when_clause ::= WHEN expr */ + case 298: /* key_opt ::= KEY expr */ yytestcase(yyruleno==298); { yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } break; - case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 278: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { assert( yymsp[-2].minor.yy473!=0 ); yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; @@ -123340,14 +129163,14 @@ static void yy_reduce( yygotominor.yy473 = yymsp[-2].minor.yy473; } break; - case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 279: /* trigger_cmd_list ::= trigger_cmd SEMI */ { assert( yymsp[-1].minor.yy473!=0 ); yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; yygotominor.yy473 = yymsp[-1].minor.yy473; } break; - case 280: /* trnm ::= nm DOT nm */ + case 281: /* trnm ::= nm DOT nm */ { yygotominor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -123355,33 +129178,33 @@ static void yy_reduce( "statements within triggers"); } break; - case 282: /* tridxby ::= INDEXED BY nm */ + case 283: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 283: /* tridxby ::= NOT INDEXED */ + case 284: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ + case 285: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ { yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } break; - case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ + case 286: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ {yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} break; - case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ + case 287: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ {yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} break; - case 287: /* trigger_cmd ::= select */ + case 288: /* trigger_cmd ::= select */ {yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } break; - case 288: /* expr ::= RAISE LP IGNORE RP */ + case 289: /* expr ::= RAISE LP IGNORE RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -123391,7 +129214,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 290: /* expr ::= RAISE LP raisetype COMMA nm RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); if( yygotominor.yy346.pExpr ) { @@ -123401,87 +129224,87 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 290: /* raisetype ::= ROLLBACK */ + case 291: /* raisetype ::= ROLLBACK */ {yygotominor.yy328 = OE_Rollback;} break; - case 292: /* raisetype ::= FAIL */ + case 293: /* raisetype ::= FAIL */ {yygotominor.yy328 = OE_Fail;} break; - case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 294: /* cmd ::= DROP TRIGGER ifexists fullname */ { sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); } break; - case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 295: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); } break; - case 295: /* cmd ::= DETACH database_kw_opt expr */ + case 296: /* cmd ::= DETACH database_kw_opt expr */ { sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); } break; - case 300: /* cmd ::= REINDEX */ + case 301: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 301: /* cmd ::= REINDEX nm dbnm */ + case 302: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 302: /* cmd ::= ANALYZE */ + case 303: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 303: /* cmd ::= ANALYZE nm dbnm */ + case 304: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 305: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); } break; - case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ + case 306: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ { sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); } break; - case 306: /* add_column_fullname ::= fullname */ + case 307: /* add_column_fullname ::= fullname */ { pParse->db->lookaside.bEnabled = 0; sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); } break; - case 309: /* cmd ::= create_vtab */ + case 310: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 310: /* cmd ::= create_vtab LP vtabarglist RP */ + case 311: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 312: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); } break; - case 314: /* vtabarg ::= */ + case 315: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 316: /* vtabargtoken ::= ANY */ - case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); - case 318: /* lp ::= LP */ yytestcase(yyruleno==318); + case 317: /* vtabargtoken ::= ANY */ + case 318: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==318); + case 319: /* lp ::= LP */ yytestcase(yyruleno==319); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 322: /* with ::= */ + case 323: /* with ::= */ {yygotominor.yy59 = 0;} break; - case 323: /* with ::= WITH wqlist */ - case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); + case 324: /* with ::= WITH wqlist */ + case 325: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==325); { yygotominor.yy59 = yymsp[0].minor.yy59; } break; - case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ + case 326: /* wqlist ::= nm eidlist_opt AS LP select RP */ { yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); } break; - case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ + case 327: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); } @@ -123509,19 +129332,19 @@ static void yy_reduce( /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); /* (91) tconscomma ::= */ yytestcase(yyruleno==91); - /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); - /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); - /* (281) tridxby ::= */ yytestcase(yyruleno==281); - /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); - /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); - /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); - /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); - /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); - /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); - /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); - /* (319) anylist ::= */ yytestcase(yyruleno==319); - /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); - /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + /* (274) foreach_clause ::= */ yytestcase(yyruleno==274); + /* (275) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==275); + /* (282) tridxby ::= */ yytestcase(yyruleno==282); + /* (299) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==299); + /* (300) database_kw_opt ::= */ yytestcase(yyruleno==300); + /* (308) kwcolumn_opt ::= */ yytestcase(yyruleno==308); + /* (309) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==309); + /* (313) vtabarglist ::= vtabarg */ yytestcase(yyruleno==313); + /* (314) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==314); + /* (316) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==316); + /* (320) anylist ::= */ yytestcase(yyruleno==320); + /* (321) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==321); + /* (322) anylist ::= anylist ANY */ yytestcase(yyruleno==322); break; }; assert( yyruleno>=0 && yyrulenoyyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -123541,13 +129364,12 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else -#endif - { + yyTraceShift(yypParser, yyact); + }else{ yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -123672,12 +129494,13 @@ SQLITE_PRIVATE void sqlite3Parser( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; yy_shift(yypParser,yyact,yymajor,&yyminorunion); yypParser->yyerrcnt--; yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); #ifdef YYERRORSYMBOL @@ -123727,7 +129550,7 @@ SQLITE_PRIVATE void sqlite3Parser( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -123777,6 +129600,11 @@ SQLITE_PRIVATE void sqlite3Parser( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } @@ -123799,6 +129627,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. */ +/* #include "sqliteInt.h" */ /* #include */ /* @@ -124163,7 +129992,11 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif + +/* Make the IdChar function accessible from ctime.c */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } +#endif /* @@ -124420,4016 +130253,23424 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } /* Otherwise fall through to the next case */ } -#endif - default: { - if( !IdChar(*z) ){ - break; - } - for(i=1; IdChar(z[i]); i++){} - *tokenType = keywordCode((char*)z, i); - return i; +#endif + default: { + if( !IdChar(*z) ){ + break; + } + for(i=1; IdChar(z[i]); i++){} + *tokenType = keywordCode((char*)z, i); + return i; + } + } + *tokenType = TK_ILLEGAL; + return 1; +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. +*/ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + int i; /* Loop counter */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ + + assert( zSql!=0 ); + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + i = 0; + assert( pzErrMsg!=0 ); + /* sqlite3ParserTrace(stdout, "parser: "); */ + pEngine = sqlite3ParserAlloc(sqlite3Malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); + enableLookaside = db->lookaside.bEnabled; + if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: { + if( db->u1.isInterrupted ){ + sqlite3ErrorMsg(pParse, "interrupt"); + pParse->rc = SQLITE_INTERRUPT; + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", + &pParse->sLastToken); + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; + } + } + } +abort_parse: + assert( nErr==0 ); + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + assert( zSql[i]==0 ); + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; + } + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } + } +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); + sqlite3_mutex_leave(sqlite3MallocMutex()); +#endif /* YYDEBUG */ + sqlite3ParserFree(pEngine, sqlite3_free); + db->lookaside.bEnabled = enableLookaside; + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); + pParse->zErrMsg = 0; + nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFree(db, p); + } + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); + } + assert( nErr==0 || pParse->rc!=SQLITE_OK ); + return nErr; +} + +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +/* #include "sqliteInt.h" */ + +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_ICU +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); + } + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SCRATCH: { + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + }else{ + pStart = pBuf; + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + db->lookaside.bMalloced = pBuf==0 ?1:0; + }else{ + db->lookaside.pStart = db; + db->lookaside.pEnd = db; + db->lookaside.bEnabled = 0; + db->lookaside.bMalloced = 0; + } +#endif /* SQLITE_OMIT_LOOKASIDE */ + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ + int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} + + +/* +** Return true if the buffer z[0..n-1] contains all spaces. +*/ +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +** +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. +*/ +static int binCollFunc( + void *padFlag, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1lastRowid; +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + HashElem *p; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( db->aDb[i].pSchema ){ + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } + } + } + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); + } + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + sqlite3VtabEponymousTableClear(db, pMod); + sqlite3DbFree(db, pMod); + } + sqlite3HashClear(&db->aModule); +#endif + + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif + + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. +*/ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_NOLFS */ "large file support is disabled", + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ "auxiliary database format error", + /* SQLITE_RANGE */ "bind or column index out of range", + /* SQLITE_NOTADB */ "file is encrypted or is not a database", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > timeout ){ + delay = timeout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + sqlite3 *db = (sqlite3 *)ptr; + int timeout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > timeout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif +} + +/* +** Invoke the given busy handler. +** +** This routine is called when an operation failed with a lock. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. +*/ +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; + rc = p->xFunc(p->pArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} + +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xFunc = xBusy; + db->busyHandler.pArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif + + +/* +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms>0 ){ + sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} + +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + db->u1.isInterrupted = 1; +} + + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + if( zFunctionName==0 || + (xFunc && (xFinal || xStep)) || + (!xFunc && (xFinal && !xStep)) || + (!xFunc && (!xFinal && xStep)) || + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + extraFlags = enc & SQLITE_DETERMINISTIC; + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Create new user functions. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + + +/* +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int nName = sqlite3Strlen30(zName); + int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Register a trace function. The pArg from the previously registered trace +** is returned. +** +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +/* +** Register a profile function. The pArg from the previously registered +** profile function is returned. +** +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_TRACE */ + +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} + +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } +#endif + return SQLITE_OK; +} + +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif +} + +/* +** Checkpoint database zDb. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } + } + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); + return 0; +#endif +} + +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM); + }else{ + testcase( db->pErr==0 ); + z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', + 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', + 'c', 'a', 'l', 'l', 'e', 'd', ' ', + 'o', 'u', 't', ' ', + 'o', 'f', ' ', + 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; + } + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode & db->errMask; +} +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode; +} + +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } + } + } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); + return SQLITE_OK; +} + + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; /* IMP: R-53341-35419 */ +} + +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc64(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + + +/* +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. +*/ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ + } + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + } + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + assert( db->pDfltColl!=0 ); + + /* Parse the filename/URI argument. */ + db->openFlags = flags; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } + sqlite3Error(db, rc); + goto opendb_out; + } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + /* The default safety_level for the main database is 'full'; for the temp + ** database it is 'NONE'. This matches the pager layer defaults. + */ + db->aDb[0].zName = "main"; + db->aDb[0].safety_level = 3; + db->aDb[1].zName = "temp"; + db->aDb[1].safety_level = 1; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterBuiltinFunctions(db); + + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + rc = sqlite3_errcode(db); + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } + } + +#ifdef SQLITE_ENABLE_FTS1 + if( !db->mallocFailed ){ + extern int sqlite3Fts1Init(sqlite3*); + rc = sqlite3Fts1Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS2 + if( !db->mallocFailed && rc==SQLITE_OK ){ + extern int sqlite3Fts2Init(sqlite3*); + rc = sqlite3Fts2Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS5 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_ICU + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3IcuInit(db); + } +#endif + +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); + } +#endif + +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbstatRegister(db); + } +#endif + +#ifdef SQLITE_ENABLE_JSON1 + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3Json1Init(db); + } +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + sqlite3_free(zOpen); + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + return rc & 0xff; +} + +/* +** Open a new database handle. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Open a new database handle. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( zFilename==0 ) zFilename = "\000\000"; + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; + } + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); +} + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ + return SQLITE_OK; +} +#endif + +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->autoCommit; +} + +/* +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They serve two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CORRUPT; +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, + "misuse at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_MISUSE; +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, + "cannot open file at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CANTOPEN; +} + + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){ +} +#endif + +/* +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } + } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = pCol->zType; + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = "BINARY"; + } + +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} + +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Invoke the xFileControl method on a particular database. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; + } + sqlite3BtreeLeave(pBtree); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ + int rc = 0; +#ifdef SQLITE_OMIT_BUILTIN_TEST + UNUSED_PARAMETER(op); +#else + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* + ** Reset the PRNG back to its uninitialized state. The next call + ** to sqlite3_randomness() will reseed the PRNG using a single call + ** to the xRandomness method of the default VFS. + */ + case SQLITE_TESTCTRL_PRNG_RESET: { + sqlite3_randomness(0,0); + break; + } + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X). + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = ALWAYS(x); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) + ** + ** Set the nReserve size to N for the main database on the database + ** connection db. + */ + case SQLITE_TESTCTRL_RESERVE: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); + sqlite3_mutex_leave(db->mutex); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + break; + } + +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + ** + ** This test control is used to create imposter tables. "db" is a pointer + ** to the database connection. dbName is the database name (ex: "main" or + ** "temp") which will receive the imposter. "onOff" turns imposter mode on + ** or off. "tnum" is the root page of the b-tree to which the imposter + ** table should connect. + ** + ** Enable imposter mode only when the schema has already been parsed. Then + ** run a single CREATE TABLE statement to construct the imposter table in + ** the parsed schema. Then turn imposter mode back off again. + ** + ** If onOff==0 and tnum>0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3_mutex_leave(db->mutex); + break; + } + } + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + return rc; +} + +/* +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 || zParam==0 ) return 0; + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + bDflt = v; + } + return bDflt; +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; + } + } + return 0; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + +/************** End of main.c ************************************************/ +/************** Begin file notify.c ******************************************/ +/* +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. +*/ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ + +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + +/* +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() +*/ + +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + +/* +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: +** +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. +** +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. +** +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. +*/ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; + + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); + + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } +} + +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; +} + +/* +** Obtain the STATIC_MASTER mutex. +*/ +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); +} + +/* +** Release the STATIC_MASTER mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); + } + } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. +*/ +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); +} + +/* +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ + + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ + + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; + } + + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; + } + } + + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ +} + +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif + +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ + +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT3 +#endif + +/* #include "sqlite3.h" */ +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +/* #include "sqlite3.h" */ + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ + +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ + +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif +}; + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); + +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ + +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3.h" */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#endif + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ + const char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ + u32 a; + +#ifndef fts3GetVarint32 + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); +#else + a = (*p++); + assert( a & 0x80 ); +#endif + + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); + a = (a & 0x0FFFFFFF ); + *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Write an error message into *pzErr +*/ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + int nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + const char *p; /* Iterator pointer */ + int nInt = 0; /* Output value */ + + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + break; + } + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} + +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + } + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; + p->bFts4 = isFts4; + p->bDescIdx = bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrzEnd ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This is safe because +** it is not possible to have a document with 2^31 terms. +*/ +#define POSITION_LIST_END 0x7fffffff + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else iCol1 = 0; + + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + assert( *p1!=POS_END && *p1!=POS_COLUMN ); + assert( *p2!=POS_END && *p2!=POS_COLUMN ); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *paOut = aOut; + *pnOut = (int)(p-aOut); + assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); + return SQLITE_OK; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut; + + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ + rc = fts3CursorSeek(0, pCsr); + + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + } + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; + return fts3SetHasStat(p); +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; + } + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + +static const sqlite3_module fts3Module = { + /* iVersion */ 2, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, + /* xClose */ fts3CloseMethod, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, + /* xColumn */ fts3ColumnMethod, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, +}; + +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ +static void hashDestroy(void *p){ + Fts3Hash *pHash = (Fts3Hash *)p; + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); +} + +/* +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. +** +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif +#ifdef SQLITE_ENABLE_ICU +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif + +/* +** Initialize the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3Hash *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif + +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif + +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); + + /* Allocate and initialize the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3Hash)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + } + + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; + } + } + +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db); + } +#endif + + /* Create the virtual table wrapper around the hash-table and overload + ** the two scalar functions. If this is successful, register the + ** module with sqlite. + */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) + ){ + rc = sqlite3_create_module_v2( + db, "fts3", &fts3Module, (void *)pHash, hashDestroy + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } + return rc; + } + + + /* An error has occurred. Delete the hash table and return the error code. */ + assert( rc!=SQLITE_OK ); + if( pHash ){ + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); + } + return rc; +} + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +*/ +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#ifdef SQLITE_TEST + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 && p->bIncr ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+1); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + if( nTmp==0 ){ + res = 0; + }else{ + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + } + + return res; +} + +/* +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif + +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_aux.c ********************************************/ +/************** Begin file fts3_expr.c ***************************************/ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +/* #include */ +/* #include */ + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(int nByte){ + void *pRet = sqlite3_malloc(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + int nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, int nNew){ + void *pRet = sqlite3_realloc(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nNear = 0; + for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ + nNear = nNear * 10 + (zInput[nKey] - '0'); + } + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +/* #include */ + +/* +** Function to query the hash-table of tokenizers (see README.tokenizers). +*/ +static int queryTestTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer_module const *pModule = 0; + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + rc = queryTestTokenizer(db, + (const char *)sqlite3_value_text(argv[0]), &pModule); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + }else if( !pModule ){ + sqlite3_result_error(context, "No such tokenizer module", -1); + goto exprtest_out; + } + + rc = pModule->xCreate(0, 0, &pTokenizer); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + pTokenizer->pModule = pModule; + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iixDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_expr.c *******************************************/ +/************** Begin file fts3_hash.c ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_hash.h" */ + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(int n){ + void *p = sqlite3_malloc(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_hash.c *******************************************/ +/************** Begin file fts3_porter.c *************************************/ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_porter.c *****************************************/ +/************** Begin file fts3_tokenizer.c **********************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void scalarFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); +} + +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST + +#include +/* #include */ + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_ANY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer.c **************************************/ +/************** Begin file fts3_tokenizer1.c *********************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ +/************** Begin file fts3_write.c **************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=FTS3_MERGE_COUNT ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nPrefix<0 || nSuffix<=0 + || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + ){ + return FTS_CORRUPT_VTAB; + } + + if( nPrefix+nSuffix>pReader->nTermAlloc ){ + int nNew = (nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( iStartLeaf<=iEndLeaf ); + if( iStartLeaf==0 ){ + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ + + for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( ppMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } + + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } + + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } + + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + + +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 ){ + rc = SQLITE_DONE; + goto finished; + } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; + + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; + + assert( csr.nSegment>0 ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevelnLeafData); + } + } + } + + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} + + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc( 10*p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_free(a); +} + +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); } - *tokenType = TK_ILLEGAL; - return 1; + + return rc; } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ - int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ - void *pEngine; /* The LEMON-generated LALR(1) parser */ - int tokenType; /* type of the next token */ - int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ - sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ - +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ - mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; - } - pParse->rc = SQLITE_OK; - pParse->zTail = zSql; - i = 0; - assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc(sqlite3Malloc); - if( pEngine==0 ){ - db->mallocFailed = 1; - return SQLITE_NOMEM; + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } - assert( pParse->pNewTable==0 ); - assert( pParse->pNewTrigger==0 ); - assert( pParse->nVar==0 ); - assert( pParse->nzVar==0 ); - assert( pParse->azVar==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; - while( !db->mallocFailed && zSql[i]!=0 ){ - assert( i>=0 ); - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - switch( tokenType ){ - case TK_SPACE: { - if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); - pParse->rc = SQLITE_INTERRUPT; - goto abort_parse; - } - break; - } - case TK_ILLEGAL: { - sqlite3DbFree(db, *pzErrMsg); - *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", - &pParse->sLastToken); - nErr++; - goto abort_parse; - } - case TK_SEMI: { - pParse->zTail = &zSql[i]; - /* Fall thru into the default case */ - } - default: { - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK ){ - goto abort_parse; - } - break; - } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); } + iRoot = 1; } -abort_parse: - if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ - if( lastTokenParsed!=TK_SEMI ){ - sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); } - sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); - } -#ifdef YYTRACKMAXSTACKDEPTH - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, - sqlite3ParserStackPeak(pEngine) - ); -#endif /* YYDEBUG */ - sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); - } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; - nErr++; - } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; - } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_free(pParse->apVtabLock); -#endif - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table - ** structure built up in pParse->pNewTable. The calling code (see vtab.c) - ** will take responsibility for freeing the Table structure. - */ - sqlite3DeleteTable(db, pParse->pNewTable); + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); - if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); - sqlite3DbFree(db, pParse->azVar); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFree(db, p); - } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); - } - if( nErr>0 && pParse->rc==SQLITE_OK ){ - pParse->rc = SQLITE_ERROR; - } - return nErr; + *pRc = rc; } -/************** End of tokenize.c ********************************************/ -/************** Begin file complete.c ****************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** An tokenizer for SQL +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. ** -** This file contains C code that implements the sqlite3_complete() API. -** This code used to be part of the tokenizer.c source file. But by -** separating it out, the code will be automatically omitted from -** static links that do not use it. +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. */ -#ifndef SQLITE_OMIT_COMPLETE +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; -/* -** This is defined in tokenize.c. We just have to import the definition. -*/ -#ifndef SQLITE_AMALGAMATION -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif -#endif /* SQLITE_AMALGAMATION */ + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + return res; +} -/* -** Token types used by the sqlite3_complete() routine. See the header -** comments on that procedure for additional information. -*/ -#define tkSEMI 0 -#define tkWS 1 -#define tkOTHER 2 -#ifndef SQLITE_OMIT_TRIGGER -#define tkEXPLAIN 3 -#define tkCREATE 4 -#define tkTEMP 5 -#define tkTRIGGER 6 -#define tkEND 7 -#endif /* -** Return TRUE if the given SQL string ends in a semicolon. -** -** Special handling is require for CREATE TRIGGER statements. -** Whenever the CREATE TRIGGER keywords are seen, the statement -** must end with ";END;". -** -** This implementation uses a state machine with 8 states: -** -** (0) INVALID We have not yet seen a non-whitespace character. -** -** (1) START At the beginning or end of an SQL statement. This routine -** returns 1 if it ends in the START state and 0 if it ends -** in any other state. -** -** (2) NORMAL We are in the middle of statement which ends with a single -** semicolon. -** -** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of -** a statement. -** -** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceded by EXPLAIN and/or followed by -** TEMP or TEMPORARY -** -** (5) TRIGGER We are in the middle of a trigger definition that must be -** ended by a semicolon, the keyword END, and another semicolon. -** -** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at -** the end of a trigger definition. +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. ** -** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger definition. +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. ** -** Transitions between states above are determined by tokens extracted -** from the input. The following tokens are significant: +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. ** -** (0) tkSEMI A semicolon. -** (1) tkWS Whitespace. -** (2) tkOTHER Any other SQL token. -** (3) tkEXPLAIN The "explain" keyword. -** (4) tkCREATE The "create" keyword. -** (5) tkTEMP The "temp" or "temporary" keyword. -** (6) tkTRIGGER The "trigger" keyword. -** (7) tkEND The "end" keyword. +** An existing segment can be appended to by an incremental merge if: ** -** Whitespace never causes a state transition and is always ignored. -** This means that a SQL string of all whitespace is invalid. +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and ** -** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed -** to recognize the end of a trigger can be omitted. All we have to do -** is look for a semicolon that is not part of an string or comment. +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. */ -SQLITE_API int sqlite3_complete(const char *zSql){ - u8 state = 0; /* Current state, using numbers defined in header comment */ - u8 token; /* Value of the next token */ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ -#ifndef SQLITE_OMIT_TRIGGER - /* A complex statement machine used to detect the end of a CREATE TRIGGER - ** statement. This is the normal case. - */ - static const u8 trans[8][8] = { - /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, - /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, - /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, - /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, - /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, - /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, - /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, - }; -#else - /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simpler - */ - static const u8 trans[3][3] = { - /* Token: */ - /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, - /* 1 START: */ { 1, 1, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, }, - }; -#endif /* SQLITE_OMIT_TRIGGER */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ - while( *zSql ){ - switch( *zSql ){ - case ';': { /* A semicolon */ - token = tkSEMI; - break; - } - case ' ': - case '\r': - case '\t': - case '\n': - case '\f': { /* White space is ignored */ - token = tkWS; - break; + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; } - case '/': { /* C-style comments */ - if( zSql[1]!='*' ){ - token = tkOTHER; - break; + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); } - zSql += 2; - while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } - if( zSql[0]==0 ) return 0; - zSql++; - token = tkWS; - break; - } - case '-': { /* SQL-style comments from "--" to end of line */ - if( zSql[1]!='-' ){ - token = tkOTHER; - break; + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; } - while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==1; - token = tkWS; - break; + nodeReaderRelease(&reader); } - case '[': { /* Microsoft-style identifiers in [...] */ - zSql++; - while( *zSql && *zSql!=']' ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } - case '`': /* Grave-accent quoted symbols used by MySQL */ - case '"': /* single- and double-quoted strings */ - case '\'': { - int c = *zSql; - zSql++; - while( *zSql && *zSql!=c ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; } - default: { -#ifdef SQLITE_EBCDIC - unsigned char c; -#endif - if( IdChar((u8)*zSql) ){ - /* Keywords and unquoted identifiers */ - int nId; - for(nId=1; IdChar(zSql[nId]); nId++){} -#ifdef SQLITE_OMIT_TRIGGER - token = tkOTHER; -#else - switch( *zSql ){ - case 'c': case 'C': { - if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ - token = tkCREATE; - }else{ - token = tkOTHER; - } - break; - } - case 't': case 'T': { - if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ - token = tkTRIGGER; - }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ - token = tkTEMP; - }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ - token = tkTEMP; - }else{ - token = tkOTHER; - } - break; - } - case 'e': case 'E': { - if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ - token = tkEND; - }else -#ifndef SQLITE_OMIT_EXPLAIN - if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ - token = tkEXPLAIN; - }else -#endif - { - token = tkOTHER; - } - break; - } - default: { - token = tkOTHER; - break; + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; } + sqlite3_free(aBlock); } -#endif /* SQLITE_OMIT_TRIGGER */ - zSql += nId-1; - }else{ - /* Operators and special symbols */ - token = tkOTHER; } - break; + nodeReaderRelease(&reader); } } - state = trans[state][token]; - zSql++; + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - return state==1; + + return rc; } -#ifndef SQLITE_OMIT_UTF16 /* -** This routine is the same as the sqlite3_complete() routine described -** above, except that the parameter is required to be UTF-16 encoded, not -** UTF-8. +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. */ -SQLITE_API int sqlite3_complete16(const void *zSql){ - sqlite3_value *pVal; - char const *zSql8; - int rc = SQLITE_NOMEM; +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zSql8 ){ - rc = sqlite3_complete(zSql8); - }else{ - rc = SQLITE_NOMEM; + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); } - sqlite3ValueFree(pVal); - return sqlite3ApiExit(0, rc); + + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_COMPLETE */ -/************** End of complete.c ********************************************/ -/************** Begin file main.c ********************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. ** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. -*/ - -#ifdef SQLITE_ENABLE_FTS3 -/************** Include fts3.h in the middle of main.c ***********************/ -/************** Begin file fts3.h ********************************************/ -/* -** 2006 Oct 10 +** In the %_segdir table, a segment is defined by the values in three +** columns: ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** start_block +** leaves_end_block +** end_block ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. ** -****************************************************************************** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. ** -** This header file is used by programs that want to link against the -** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; -/************** End of fts3.h ************************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_RTREE -/************** Include rtree.h in the middle of main.c **********************/ -/************** Begin file rtree.h *******************************************/ -/* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** RTREE library. All it does is declare the sqlite3RtreeInit() interface. -*/ + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; -#if 0 -extern "C" { -#endif /* __cplusplus */ + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} -/************** End of rtree.h ***********************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_ICU -/************** Include sqliteicu.h in the middle of main.c ******************/ -/************** Begin file sqliteicu.h ***************************************/ /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Remove an entry from the %_segdir table. This involves running the +** following two statements: ** -****************************************************************************** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx ** -** This header file is used by programs that want to link against the -** ICU extension. All it does is declare the sqlite3IcuInit() interface. +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. */ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ - - -/************** End of sqliteicu.h *******************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } -#ifndef SQLITE_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. -*/ -SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; -#endif + return rc; +} -/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. */ -SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ -/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a -** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. -*/ -SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } -/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function -** returns an integer equal to SQLITE_VERSION_NUMBER. -*/ -SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } -/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled with mutexing code omitted due to -** the SQLITE_THREADSAFE compile-time option being set to 0. -*/ -SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) -/* -** If the following function pointer is not NULL and if -** SQLITE_ENABLE_IOTRACE is enabled, then messages describing -** I/O active are written using this function. These messages -** are intended for debugging activity only. -*/ -SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; -#endif + sqlite3_free(aIdx); + return rc; +} -/* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** temporary files. -** -** See also the "PRAGMA temp_store_directory" SQL command. -*/ -SQLITE_API char *sqlite3_temp_directory = 0; +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} /* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** all database files specified with a relative pathname. +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. ** -** See also the "PRAGMA data_store_directory" SQL command. +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. */ -SQLITE_API char *sqlite3_data_directory = 0; +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ -/* -** Initialize SQLite. -** -** This routine must be called to initialize the memory allocation, -** VFS, and mutex subsystems prior to doing any serious work with -** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite3_open(). -** -** This routine is a no-op except on its very first call for the process, -** or for the first call after a call to sqlite3_shutdown. -** -** The first thread to call this routine runs the initialization to -** completion. If subsequent threads call this routine before the first -** thread has finished the initialization process, then the subsequent -** threads must block until the first thread finishes with the initialization. -** -** The first thread might call this routine recursively. Recursive -** calls to this routine should not block, of course. Otherwise the -** initialization process would never complete. -** -** Let X be the first thread to enter this routine. Let Y be some other -** thread. Then while the initial invocation of this routine by X is -** incomplete, it is required that: -** -** * Calls to this routine from Y must block until the outer-most -** call by X completes. -** -** * Recursive calls to this routine from thread X return immediately -** without blocking. -*/ -SQLITE_API int sqlite3_initialize(void){ - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ - int rc; /* Result code */ -#ifdef SQLITE_EXTRA_INIT - int bRunExtraInit = 0; /* Extra initialization needed */ -#endif + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; -#ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; } -#endif + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); - /* If SQLite is already completely initialized, then this call - ** to sqlite3_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} - /* Make sure the mutex subsystem is initialized. If unable to - ** initialize the mutex subsystem, return early with the error. - ** If the system is so sick that we are unable to allocate a mutex, - ** there is not much SQLite is going to be able to do. - ** - ** The mutex subsystem must take care of serializing its own - ** initialization. - */ - rc = sqlite3MutexInit(); - if( rc ) return rc; +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ - /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that - ** MutexAlloc() is called for a static mutex prior to initializing the - ** malloc subsystem - this implies that the allocation of a static - ** mutex must not require support from the malloc subsystem. - */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.isMutexInit = 1; - if( !sqlite3GlobalConfig.isMallocInit ){ - rc = sqlite3MallocInit(); - } + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isMallocInit = 1; - if( !sqlite3GlobalConfig.pInitMutex ){ - sqlite3GlobalConfig.pInitMutex = - sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; - } + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.nRefInitMutex++; - } - sqlite3_mutex_leave(pMaster); - /* If rc is not SQLITE_OK at this point, then either the malloc - ** subsystem could not be initialized or the system failed to allocate - ** the pInitMutex mutex. Return an error in either case. */ - if( rc!=SQLITE_OK ){ - return rc; - } + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; - /* Do the rest of the initialization under the recursive mutex so - ** that we will be able to handle recursive calls into - ** sqlite3_initialize(). The recursive calls normally come through - ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other - ** recursive calls might also be possible. - ** - ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls - ** to the xInit method, so the xInit method need not be threadsafe. - ** - ** The following mutex is what serializes access to the appdef pcache xInit - ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the - ** call to sqlite3PcacheInitialize(). - */ - sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); - if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - sqlite3GlobalConfig.inProgress = 1; - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); - if( sqlite3GlobalConfig.isPCacheInit==0 ){ - rc = sqlite3PcacheInitialize(); + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); } if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isPCacheInit = 1; - rc = sqlite3OsInit(); + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, - sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); - sqlite3GlobalConfig.isInit = 1; -#ifdef SQLITE_EXTRA_INIT - bRunExtraInit = 1; -#endif + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); } - sqlite3GlobalConfig.inProgress = 0; } - sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); - /* Go back under the static mutex and clean up the recursive - ** mutex to prevent a resource leak. - */ - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.nRefInitMutex--; - if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ - assert( sqlite3GlobalConfig.nRefInitMutex==0 ); - sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); - sqlite3GlobalConfig.pInitMutex = 0; + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } } - sqlite3_mutex_leave(pMaster); - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite3IsNaN(y) ); + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } } -#endif -#endif - /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT - ** compile-time option. - */ -#ifdef SQLITE_EXTRA_INIT - if( bRunExtraInit ){ - int SQLITE_EXTRA_INIT(const char*); - rc = SQLITE_EXTRA_INIT(0); + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); } -#endif + *pnRem = nRem; return rc; } /* -** Undo the effects of sqlite3_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. +** Store an incr-merge hint in the database. */ -SQLITE_API int sqlite3_shutdown(void){ - if( sqlite3GlobalConfig.isInit ){ -#ifdef SQLITE_EXTRA_SHUTDOWN - void SQLITE_EXTRA_SHUTDOWN(void); - SQLITE_EXTRA_SHUTDOWN(); -#endif - sqlite3_os_end(); - sqlite3_reset_auto_extension(); - sqlite3GlobalConfig.isInit = 0; - } - if( sqlite3GlobalConfig.isPCacheInit ){ - sqlite3PcacheShutdown(); - sqlite3GlobalConfig.isPCacheInit = 0; +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); } - if( sqlite3GlobalConfig.isMallocInit ){ - sqlite3MallocEnd(); - sqlite3GlobalConfig.isMallocInit = 0; -#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - /* The heap subsystem has now been shutdown and these values are supposed - ** to be NULL or point to memory that was obtained from sqlite3_malloc(), - ** which would rely on that heap subsystem; therefore, make sure these - ** values cannot refer to heap memory that was just invalidated when the - ** heap subsystem was shutdown. This is only done if the current call to - ** this function resulted in the heap subsystem actually being shutdown. - */ - sqlite3_data_directory = 0; - sqlite3_temp_directory = 0; -#endif + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - if( sqlite3GlobalConfig.isMutexInit ){ - sqlite3MutexEnd(); - sqlite3GlobalConfig.isMutexInit = 0; + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return FTS_CORRUPT_VTAB; return SQLITE_OK; } + /* -** This API allows applications to modify the global configuration of -** the SQLite library at run-time. +** Attempt an incremental merge that writes nMerge leaf blocks. ** -** This routine should only be called when there are no outstanding -** database connections or memory allocations. This routine is not -** threadsafe. Failure to heed these warnings can lead to unpredictable -** behavior. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ -SQLITE_API int sqlite3_config(int op, ...){ - va_list ap; - int rc = SQLITE_OK; +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; - va_start(ap, op); - switch( op ){ + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - /* Mutex configuration options are only available in a threadsafe - ** compile. + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 - case SQLITE_CONFIG_SINGLETHREAD: { - /* Disable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 0; - sqlite3GlobalConfig.bFullMutex = 0; - break; - } - case SQLITE_CONFIG_MULTITHREAD: { - /* Disable mutexing of database connections */ - /* Enable mutexing of core data structures */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 0; - break; - } - case SQLITE_CONFIG_SERIALIZED: { - /* Enable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 1; - break; - } - case SQLITE_CONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - break; - } - case SQLITE_CONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; - break; + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; } -#endif + rc = sqlite3_reset(pFindLevel); + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ - case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ - sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); - break; - } - case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ - if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); - *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; - break; - } - case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ - sqlite3GlobalConfig.pPage = va_arg(ap, void*); - sqlite3GlobalConfig.szPage = va_arg(ap, int); - sqlite3GlobalConfig.nPage = va_arg(ap, int); - break; + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } } - case SQLITE_CONFIG_PCACHE: { - /* no-op */ - break; - } - case SQLITE_CONFIG_GETPCACHE: { - /* now an error */ - rc = SQLITE_ERROR; - break; + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } } - case SQLITE_CONFIG_PCACHE2: { - /* Specify an alternative page cache implementation */ - sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); - break; + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } - case SQLITE_CONFIG_GETPCACHE2: { - if( sqlite3GlobalConfig.pcache2.xInit==0 ){ - sqlite3PCacheSetDefault(); + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } - *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; - break; - } -#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) - case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ - sqlite3GlobalConfig.pHeap = va_arg(ap, void*); - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - sqlite3GlobalConfig.mnReq = va_arg(ap, int); + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); - if( sqlite3GlobalConfig.mnReq<1 ){ - sqlite3GlobalConfig.mnReq = 1; - }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ - /* cap min request size at 2^12 */ - sqlite3GlobalConfig.mnReq = (1<<12); + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } } - if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. - */ - memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); - }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. The enclosing #if guarantees at - ** least one of these methods is currently enabled. - */ -#ifdef SQLITE_ENABLE_MEMSYS3 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); -#endif + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); } - break; } -#endif - case SQLITE_CONFIG_LOOKASIDE: { - sqlite3GlobalConfig.szLookaside = va_arg(ap, int); - sqlite3GlobalConfig.nLookaside = va_arg(ap, int); - break; - } - - /* Record a pointer to the logger function and its first argument. - ** The default is NULL. Logging is disabled if the function pointer is - ** NULL. - */ - case SQLITE_CONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); - break; - } + sqlite3Fts3SegReaderFinish(pCsr); + } - /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames - ** can be changed at start-time using the - ** sqlite3_config(SQLITE_CONFIG_URI,1) or - ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. - */ - case SQLITE_CONFIG_URI: { - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); - break; - } + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } - case SQLITE_CONFIG_COVERING_INDEX_SCAN: { - sqlite3GlobalConfig.bUseCis = va_arg(ap, int); - break; - } + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} -#ifdef SQLITE_ENABLE_SQLLOG - case SQLITE_CONFIG_SQLLOG: { - typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); - sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); - sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); - break; - } -#endif +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} - case SQLITE_CONFIG_MMAP_SIZE: { - sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); - sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - sqlite3GlobalConfig.mxMmap = mxMmap; - if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; - if( szMmap>mxMmap) szMmap = mxMmap; - sqlite3GlobalConfig.szMmap = szMmap; - break; - } +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) - case SQLITE_CONFIG_WIN32_HEAPSIZE: { - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - break; - } -#endif + /* Read the first integer value */ + nMerge = fts3Getint(&z); - default: { - rc = SQLITE_ERROR; - break; + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); } + sqlite3Fts3SegmentsClose(p); } - va_end(ap); return rc; } /* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. +** Process statements of the form: ** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ - void *pStart; - if( db->lookaside.nOut ){ - return SQLITE_BUSY; - } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. - */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; - }else{ - pStart = pBuf; +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; } - db->lookaside.pStart = pStart; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; - p = (LookasideSlot*)&((u8*)p)[sz]; - } - db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; - db->lookaside.bMalloced = pBuf==0 ?1:0; - }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; - db->lookaside.bEnabled = 0; - db->lookaside.bMalloced = 0; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; } - return SQLITE_OK; + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; } /* -** Return the mutex associated with a database connection. +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. */ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ - return db->mutex; +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; imutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - Pager *pPager = sqlite3BtreePager(pBt); - sqlite3PagerShrink(pPager); +static u64 fts3ChecksumIndex( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id to return cksum for */ + int iIndex, /* Index to cksum (0..p->nIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrmutex); - return SQLITE_OK; + sqlite3Fts3SegReaderFinish(&csr); + + *pRc = rc; + return cksum; } /* -** Configuration settings for an individual database connection +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. */ -SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ - va_list ap; - int rc; - va_start(ap, op); - switch( op ){ - case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ - int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ - int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); - break; +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } } - default: { - static const struct { - int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ - } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, - }; - unsigned int i; - rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ - for(i=0; iflags; - if( onoff>0 ){ - db->flags |= aFlagOp[i].mask; - }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; - } - if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); - } - if( pRes ){ - *pRes = (db->flags & aFlagOp[i].mask)!=0; + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } } - rc = SQLITE_OK; - break; + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } } - break; } - } - va_end(ap); - return rc; -} + sqlite3_finalize(pStmt); + } -/* -** Return true if the buffer z[0..n-1] contains all spaces. -*/ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; + *pbOk = (cksum1==cksum2); + return rc; } /* -** This is the default collating function named "BINARY" which is always -** available. +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. ** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. */ -static int binCollFunc( - void *padFlag, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ ){ - int rc, n; - n = nKey1)" ** -** At the moment there is only a UTF-8 implementation. +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. */ -static int nocaseCollatingFunc( - void *NotUsed, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - int r = sqlite3StrNICmp( - (const char *)pKey1, (const char *)pKey2, (nKey1lastRowid; -} + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#ifdef SQLITE_TEST + }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + p->nNodeSize = atoi(&zVal[9]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; +#endif + }else{ + rc = SQLITE_ERROR; + } -/* -** Return the number of changes in the most recent call to sqlite3_exec(). -*/ -SQLITE_API int sqlite3_changes(sqlite3 *db){ - return db->nChange; + return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* -** Return the number of changes since the database handle was opened. +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. */ -SQLITE_API int sqlite3_total_changes(sqlite3 *db){ - return db->nTotalChange; +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } } /* -** Close all open savepoints. This function only manipulates fields of the -** database handle object, it does not close any savepoints that may be open -** at the b-tree/pager level. +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). */ -SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ - while( db->pSavepoint ){ - Savepoint *pTmp = db->pSavepoint; - db->pSavepoint = pTmp->pNext; - sqlite3DbFree(db, pTmp); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); } - db->nSavepoint = 0; - db->nStatement = 0; - db->isTransactionSavepoint = 0; + pCsr->pDeferred = 0; } /* -** Invoke the destructor function associated with FuncDef p, if any. Except, -** if this is not the last copy of the function, do not invoke it. Multiple -** copies of a single function are created when create_function() is called -** with SQLITE_ANY as the encoding. +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. */ -static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; - if( pDestructor ){ - pDestructor->nRef--; - if( pDestructor->nRef==0 ){ - pDestructor->xDestroy(pDestructor->pUserData); - sqlite3DbFree(db, pDestructor); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } } - } -} -/* -** Disconnect all sqlite3_vtab objects that belong to database connection -** db. This is called when db is being closed. -*/ -static void disconnectAllVtab(sqlite3 *db){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - int i; - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ - HashElem *p; - for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ - Table *pTab = (Table *)sqliteHashData(p); - if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); } } } - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); -#else - UNUSED_PARAMETER(db); -#endif -} -/* -** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. -*/ -static int connectionIsBusy(sqlite3 *db){ - int j; - assert( sqlite3_mutex_held(db->mutex) ); - if( db->pVdbe ) return 1; - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; - } - return 0; + return rc; } -/* -** Close an existing SQLite database -*/ -static int sqlite3Close(sqlite3 *db, int forceZombie){ - if( !db ){ - /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or - ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ return SQLITE_OK; } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - sqlite3_mutex_enter(db->mutex); - /* Force xDisconnect calls on all virtual tables */ - disconnectAllVtab(db); + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; - /* If a transaction is open, the disconnectAllVtab() call above - ** will not have called the xDisconnect() method on any virtual - ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() - ** call will do so. We need to do this before the check for active - ** SQL statements below, as the v-table implementation may be storing - ** some prepared statements internally. - */ - sqlite3VtabRollback(db); + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} - /* Legacy behavior (sqlite3_close() behavior) is to return - ** SQLITE_BUSY if the connection can not be closed immediately. - */ - if( !forceZombie && connectionIsBusy(db) ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " - "statements or unfinished backups"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Closing the handle. Fourth parameter is passed the value 2. */ - sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); - } -#endif + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; - /* Convert the connection into a zombie and then close it. - */ - db->magic = SQLITE_MAGIC_ZOMBIE; - sqlite3LeaveMutexAndCloseZombie(db); return SQLITE_OK; } +#endif /* -** Two variations on the public interface for closing a database -** connection. The sqlite3_close() version returns SQLITE_BUSY and -** leaves the connection option if there are unfinalized prepared -** statements or unfinished sqlite3_backups. The sqlite3_close_v2() -** version forces the connection to become a zombie if there are -** unclosed resources, and arranges for deallocation when the last -** prepare statement or sqlite3_backup closes. +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. */ -SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } -SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + } + return rc; +} /* -** Close the mutex on database connection db. +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: ** -** Furthermore, if database connection db is a zombie (meaning that there -** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and -** every sqlite3_stmt has now been finalized and every sqlite3_backup has -** finished, then free all resources. +** CREATE TABLE
    ( +** , +**
    HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** */ -SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; - - /* If there are outstanding sqlite3_stmt or sqlite3_backup objects - ** or if the connection has not yet been closed by sqlite3_close_v2(), - ** then just leave the mutex and return. - */ - if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ - sqlite3_mutex_leave(db->mutex); - return; - } +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + int isRemove = 0; /* True for an UPDATE or DELETE */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; - /* If we reach this point, it means that the database connection has - ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** passed to sqlite3_close (meaning that it is a zombie). Therefore, - ** go ahead and free all resources. - */ + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); - /* If a transaction is open, roll it back. This also ensures that if - ** any database schemas have been modified by an uncommitted transaction - ** they are reset. And that the required b-tree mutex is held to make - ** the pager rollback and schema reset an atomic operation. */ - sqlite3RollbackAll(db, SQLITE_OK); + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); - /* Free any outstanding Savepoint structures. */ - sqlite3CloseSavepoints(db); + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } - /* Close all database connections */ - for(j=0; jnDb; j++){ - struct Db *pDb = &db->aDb[j]; - if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 ){ - pDb->pSchema = 0; - } - } + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; } - /* Clear the TEMP schema separately and last */ - if( db->aDb[1].pSchema ){ - sqlite3SchemaClear(db->aDb[1].pSchema); + + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; } - sqlite3VtabUnlockList(db); + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); - /* Free up the array of auxiliary databases */ - sqlite3CollapseDatabaseArray(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; - /* Tell the code in notify.c that the connection no longer holds any - ** locks and does not require any further unlock-notify callbacks. + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. */ - sqlite3ConnectionClosed(db); - - for(j=0; jaFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; } - } - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(i); - /* Invoke any destructors registered for collation sequence user data. */ - for(j=0; j<3; j++){ - if( pColl[j].xDel ){ - pColl[j].xDel(pColl[j].pUser); + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; } } - sqlite3DbFree(db, pColl); } - sqlite3HashClear(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ - Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } - sqlite3DbFree(db, pMod); + if( rc!=SQLITE_OK ){ + goto update_out; } - sqlite3HashClear(&db->aModule); -#endif - - sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ - sqlite3ValueFree(db->pErr); - sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); -#endif - - db->magic = SQLITE_MAGIC_ERROR; - /* The temp-database schema is allocated differently from the other schema - ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). - ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database - ** structure? - */ - sqlite3DbFree(db, db->aDb[1].pSchema); - sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; - sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; } - sqlite3_free(db); -} - -/* -** Rollback all database files. If tripCode is not SQLITE_OK, then -** any write cursors are invalidated ("tripped" - as in "tripping a circuit -** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. Read cursors remain open and valid -** but are "saved" in case the table pages are moved around. -*/ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ - int i; - int inTrans = 0; - int schemaChange; - assert( sqlite3_mutex_held(db->mutex) ); - sqlite3BeginBenignMalloc(); - - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). - ** This is important in case the transaction being rolled back has - ** modified the database schema. If the b-tree mutexes are not taken - ** here, then another shared-cache connection might sneak in between - ** the database rollback and schema reset, which can cause false - ** corruption reports in some cases. */ - sqlite3BtreeEnterAll(db); - schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; - - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p ){ - if( sqlite3BtreeIsInTrans(p) ){ - inTrans = 1; + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; } - sqlite3BtreeRollback(p, tripCode, !schemaChange); } + if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; } - sqlite3VtabRollback(db); - sqlite3EndBenignMalloc(); - if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetAllSchemasOfConnection(db); + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } - sqlite3BtreeLeaveAll(db); - - /* Any deferred constraint violations have now been resolved. */ - db->nDeferredCons = 0; - db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; - /* If one has been configured, invoke the rollback-hook callback */ - if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ - db->xRollbackCallback(db->pRollbackArg); - } + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; } -/* -** Return a static string containing the name corresponding to the error code -** specified in the argument. +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. */ -#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ - const char *zName = 0; - int i, origRc = rc; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; - case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; - case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; - case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; - case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; - case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; - case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; - case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; - case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; - case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; - case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; - case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; - case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; - case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; - case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: - zName = "SQLITE_CONSTRAINT_FUNCTION"; break; - case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } - if( zName==0 ){ - static char zBuf[50]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); - zName = zBuf; - } - return zName; + sqlite3Fts3SegmentsClose(p); + return rc; } + #endif +/************** End of fts3_write.c ******************************************/ +/************** Begin file fts3_snippet.c ************************************/ /* -** Return a static string that describes the kind of error specified in the -** argument. +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** */ -SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ - static const char* const aMsg[] = { - /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error or missing database", - /* SQLITE_INTERNAL */ 0, - /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "callback requested query abort", - /* SQLITE_BUSY */ "database is locked", - /* SQLITE_LOCKED */ "database table is locked", - /* SQLITE_NOMEM */ "out of memory", - /* SQLITE_READONLY */ "attempt to write a readonly database", - /* SQLITE_INTERRUPT */ "interrupted", - /* SQLITE_IOERR */ "disk I/O error", - /* SQLITE_CORRUPT */ "database disk image is malformed", - /* SQLITE_NOTFOUND */ "unknown operation", - /* SQLITE_FULL */ "database or disk is full", - /* SQLITE_CANTOPEN */ "unable to open database file", - /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ "table contains no data", - /* SQLITE_SCHEMA */ "database schema has changed", - /* SQLITE_TOOBIG */ "string or blob too big", - /* SQLITE_CONSTRAINT */ "constraint failed", - /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "library routine called out of sequence", - /* SQLITE_NOLFS */ "large file support is disabled", - /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ "auxiliary database format error", - /* SQLITE_RANGE */ "bind or column index out of range", - /* SQLITE_NOTADB */ "file is encrypted or is not a database", - }; - const char *zErr = "unknown error"; - switch( rc ){ - case SQLITE_ABORT_ROLLBACK: { - zErr = "abort due to ROLLBACK"; - break; - } - default: { - rc &= 0xff; - if( ALWAYS(rc>=0) && rc */ +/* #include */ /* -** This routine implements a busy callback that sleeps and tries -** again until a timeout value is reached. The timeout value is -** an integer number of milliseconds passed in as the first -** argument. +** Characters that may appear in the second argument to matchinfo(). */ -static int sqliteDefaultBusyCallback( - void *ptr, /* Database connection */ - int count /* Number of times table has been busy */ -){ -#if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) - static const u8 delays[] = - { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; - static const u8 totals[] = - { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; -# define NDELAY ArraySize(delays) - sqlite3 *db = (sqlite3 *)ptr; - int timeout = db->busyTimeout; - int delay, prior; - - assert( count>=0 ); - if( count < NDELAY ){ - delay = delays[count]; - prior = totals[count]; - }else{ - delay = delays[NDELAY-1]; - prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); - } - if( prior + delay > timeout ){ - delay = timeout - prior; - if( delay<=0 ) return 0; - } - sqlite3OsSleep(db->pVfs, delay*1000); - return 1; -#else - sqlite3 *db = (sqlite3 *)ptr; - int timeout = ((sqlite3 *)ptr)->busyTimeout; - if( (count+1)*1000 > timeout ){ - return 0; - } - sqlite3OsSleep(db->pVfs, 1000000); - return 1; -#endif -} +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ /* -** Invoke the given busy handler. -** -** This routine is called when an operation failed with a lock. -** If this routine returns non-zero, the lock is retried. If it -** returns 0, the operation aborts with an SQLITE_BUSY error. +** The default value for the second argument to matchinfo(). */ -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ - int rc; - if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; - rc = p->xFunc(p->pArg, p->nBusy); - if( rc==0 ){ - p->nBusy = -1; - }else{ - p->nBusy++; - } - return rc; -} +#define FTS3_MATCHINFO_DEFAULT "pcx" + /* -** This routine sets the busy callback for an Sqlite database to the -** given callback function with the given argument. +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. */ -SQLITE_API int sqlite3_busy_handler( - sqlite3 *db, - int (*xBusy)(void*,int), - void *pArg -){ - sqlite3_mutex_enter(db->mutex); - db->busyHandler.xFunc = xBusy; - db->busyHandler.pArg = pArg; - db->busyHandler.nBusy = 0; - db->busyTimeout = 0; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* -** This routine sets the progress callback for an Sqlite database to the -** given callback function with the given argument. The progress callback will -** be invoked every nOps opcodes. +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. */ -SQLITE_API void sqlite3_progress_handler( - sqlite3 *db, - int nOps, - int (*xProgress)(void*), - void *pArg -){ - sqlite3_mutex_enter(db->mutex); - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = (unsigned)nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; - } - sqlite3_mutex_leave(db->mutex); -} -#endif +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; /* -** This routine installs a default busy handler that waits for the -** specified number of milliseconds before returning 0. +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. */ -SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ - if( ms>0 ){ - sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); - db->busyTimeout = ms; - }else{ - sqlite3_busy_handler(db, 0, 0); - } - return SQLITE_OK; -} +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; /* -** Cause any pending operation to stop at its earliest opportunity. +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. */ -SQLITE_API void sqlite3_interrupt(sqlite3 *db){ - db->u1.isInterrupted = 1; -} +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; + + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + /* -** This function is exactly the same as sqlite3_create_function(), except -** that it is designed to be called by internal code. The difference is -** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** Allocate a two-slot MatchinfoBuffer object. */ -SQLITE_PRIVATE int sqlite3CreateFunc( - sqlite3 *db, - const char *zFunctionName, - int nArg, - int enc, - void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - FuncDestructor *pDestructor -){ - FuncDef *p; - int nName; - int extraFlags; +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); + int nStr = (int)strlen(zMatchinfo); - assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE_BKPT; + pRet = sqlite3_malloc(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); + pRet->nElem = nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; } - assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); - extraFlags = enc & SQLITE_DETERMINISTIC; - enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); - -#ifndef SQLITE_OMIT_UTF16 - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - ** - ** If SQLITE_ANY is specified, add three versions of the function - ** to the hash table. - */ - if( enc==SQLITE_UTF16 ){ - enc = SQLITE_UTF16NATIVE; - }else if( enc==SQLITE_ANY ){ - int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); - if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); - } - if( rc!=SQLITE_OK ){ - return rc; - } - enc = SQLITE_UTF16BE; - } -#else - enc = SQLITE_UTF8; -#endif - - /* Check if an existing function is being overridden or deleted. If so, - ** and there are active VMs, then return SQLITE_BUSY. If a function - ** is being overridden/deleted but there are no active VMs, allow the - ** operation to continue but invalidate all precompiled statements. - */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); - if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); - return SQLITE_BUSY; - }else{ - sqlite3ExpirePreparedStatements(db); - } + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); - assert(p || db->mallocFailed); - if( !p ){ - return SQLITE_NOMEM; + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); } +} - /* If an older version of the function with a configured destructor is - ** being replaced invoke the destructor function here. */ - functionDestroy(db, p); +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; - if( pDestructor ){ - pDestructor->nRef++; + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; } - p->pDestructor = pDestructor; - p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; - testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xFunc = xFunc; - p->xStep = xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = (u16)nArg; - return SQLITE_OK; + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; } -/* -** Create new user functions. -*/ -SQLITE_API int sqlite3_create_function( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, - xFinal, 0); +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); } -SQLITE_API int sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) -){ - int rc = SQLITE_ERROR; - FuncDestructor *pArg = 0; - sqlite3_mutex_enter(db->mutex); - if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); - if( !pArg ){ - xDestroy(p); - goto out; +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); } - pArg->xDestroy = xDestroy; - pArg->pUserData = p; - } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); - if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); - xDestroy(p); - sqlite3DbFree(db, pArg); } - - out: - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; } -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_create_function16( - sqlite3 *db, - const void *zFunctionName, - int nArg, - int eTextRep, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), - void (*xStep)(sqlite3_context*,int,sqlite3_value**), - void (*xFinal)(sqlite3_context*) -){ - int rc; - char *zFunc8; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); - sqlite3DbFree(db, zFunc8); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif +/* +** End of MatchinfoBuffer code. +*************************************************************************/ /* -** Declare that a function has been overloaded by a virtual table. +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: ** -** If the function already exists as a regular global function, then -** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. +** 4 9 113 ** -** When virtual tables intend to provide an overloaded function, they -** should call this routine to make sure the global function exists. -** A global function must exist in order for name resolution to work -** properly. +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. */ -SQLITE_API int sqlite3_overload_function( - sqlite3 *db, - const char *zName, - int nArg +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ ){ - int nName = sqlite3Strlen30(zName); - int rc = SQLITE_OK; - sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ - rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); return rc; } -#ifndef SQLITE_OMIT_TRACE /* -** Register a trace function. The pArg from the previously registered trace -** is returned. +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. ** -** A NULL trace function means that no tracing is executes. A non-NULL -** trace is a pointer to a function that is invoked at the start of each -** SQL statement. +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. */ -SQLITE_API void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ - void *pOld; - sqlite3_mutex_enter(db->mutex); - pOld = db->pTraceArg; - db->xTrace = xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } + + /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. -** -** A NULL profile function means that no profiling is executes. A non-NULL -** profile is a pointer to a function that is invoked at the conclusion of -** each SQL statement that is run. +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). */ -SQLITE_API void *sqlite3_profile( - sqlite3 *db, - void (*xProfile)(void*,const char*,sqlite_uint64), - void *pArg -){ - void *pOld; - sqlite3_mutex_enter(db->mutex); - pOld = db->pProfileArg; - db->xProfile = xProfile; - db->pProfileArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; + + return rc; } -#endif /* SQLITE_OMIT_TRACE */ /* -** Register a function to be invoked when a transaction commits. -** If the invoked function returns non-zero, then the commit becomes a -** rollback. +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. */ -SQLITE_API void *sqlite3_commit_hook( - sqlite3 *db, /* Attach the hook to this database */ - int (*xCallback)(void*), /* Function to invoke on each commit */ - void *pArg /* Argument to the function */ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ ){ - void *pOld; - sqlite3_mutex_enter(db->mutex); - pOld = db->pCommitArg; - db->xCommitCallback = xCallback; - db->pCommitArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; } /* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. */ -SQLITE_API void *sqlite3_update_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), - void *pArg /* Argument to the function */ -){ - void *pRet; - sqlite3_mutex_enter(db->mutex); - pRet = db->pUpdateArg; - db->xUpdateCallback = xCallback; - db->pUpdateArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; + + while( iItermutex); - pRet = db->pRollbackArg; - db->xRollbackCallback = xCallback; - db->pRollbackArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; +static int fts3SnippetNextCandidate(SnippetIter *pIter){ + int i; /* Loop counter */ + + if( pIter->iCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } + } + + return 0; } -#ifndef SQLITE_OMIT_WAL /* -** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). -** Invoke sqlite3_wal_checkpoint if the number of frames in the log file -** is greater than sqlite3.pWalArg cast to an integer (the value configured by -** wal_autocheckpoint()). -*/ -SQLITE_PRIVATE int sqlite3WalDefaultHook( - void *pClientData, /* Argument */ - sqlite3 *db, /* Connection */ - const char *zDb, /* Database */ - int nFrame /* Size of WAL */ +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ ){ - if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ - sqlite3BeginBenignMalloc(); - sqlite3_wal_checkpoint(db, zDb); - sqlite3EndBenignMalloc(); + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) ){ + int j; + u64 mPhrase = (u64)1 << i; + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } } - return SQLITE_OK; + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; } -#endif /* SQLITE_OMIT_WAL */ /* -** Configure an sqlite3_wal_hook() callback to automatically checkpoint -** a database after committing a transaction if there are nFrame or -** more frames in the log file. Passing zero or a negative value as the -** nFrame parameter disables automatic checkpoints entirely. -** -** The callback registered by this function replaces any existing callback -** registered using sqlite3_wal_hook(). Likewise, registering a callback -** using sqlite3_wal_hook() disables the automatic checkpoint mechanism -** configured by this function. +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(nFrame); -#else - if( nFrame>0 ){ - sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; + + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; }else{ - sqlite3_wal_hook(db, 0, 0); + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); } -#endif - return SQLITE_OK; -} -/* -** Register a callback to be invoked each time a transaction is written -** into the write-ahead-log by this database connection. -*/ -SQLITE_API void *sqlite3_wal_hook( - sqlite3 *db, /* Attach the hook to this db handle */ - int(*xCallback)(void *, sqlite3*, const char*, int), - void *pArg /* First argument passed to xCallback() */ -){ -#ifndef SQLITE_OMIT_WAL - void *pRet; - sqlite3_mutex_enter(db->mutex); - pRet = db->pWalArg; - db->xWalCallback = xCallback; - db->pWalArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -#else - return 0; -#endif + return rc; } /* -** Checkpoint database zDb. +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. */ -SQLITE_API int sqlite3_wal_checkpoint_v2( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of attached database (or NULL) */ - int eMode, /* SQLITE_CHECKPOINT_* value */ - int *pnLog, /* OUT: Size of WAL log in frames */ - int *pnCkpt /* OUT: Total number of frames checkpointed */ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ ){ -#ifdef SQLITE_OMIT_WAL - return SQLITE_OK; -#else - int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + int nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ - /* Initialize the output variables to -1 in case an error occurs. */ - if( pnLog ) *pnLog = -1; - if( pnCkpt ) *pnCkpt = -1; + memset(&sIter, 0, sizeof(sIter)); - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ - return SQLITE_MISUSE; + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; } - sqlite3_mutex_enter(db->mutex); - if( zDb && zDb[0] ){ - iDb = sqlite3FindDbName(db, zDb); + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; } - if( iDb<0 ){ - rc = SQLITE_ERROR; - sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); - }else{ - rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc); + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + *piScore = iBestScore; } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); + sqlite3_free(sIter.aPhrase); return rc; -#endif } /* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + int nAlloc = pStr->nAlloc+nAppend+100; + char *zNew = sqlite3_realloc(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; } -#ifndef SQLITE_OMIT_WAL /* -** Run a checkpoint on database iDb. This is a no-op if database iDb is -** not currently open in WAL mode. +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: ** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is -** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** ........X.....X ** -** The mutex on database handle db should be held by the caller. The mutex -** associated with the specific b-tree being checkpointed is taken by -** this function while the checkpoint is running. +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: ** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are -** checkpointed. If an error is encountered it is returned immediately - -** no attempt is made to checkpoint any remaining databases. +** ....X.....X.... ** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. */ -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterate through attached dbs */ - int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ - assert( sqlite3_mutex_held(db->mutex) ); - assert( !pnLog || *pnLog==-1 ); - assert( !pnCkpt || *pnCkpt==-1 ); + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ - for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); - pnLog = 0; - pnCkpt = 0; - if( rc==SQLITE_BUSY ){ - bBusy = 1; - rc = SQLITE_OK; + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; } } } - - return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; -} -#endif /* SQLITE_OMIT_WAL */ - -/* -** This function returns true if main-memory should be used instead of -** a temporary file for transient pager files and statement journals. -** The value returned depends on the value of db->temp_store (runtime -** parameter) and the compile time value of SQLITE_TEMP_STORE. The -** following table describes the relationship between these two values -** and this functions return value. -** -** SQLITE_TEMP_STORE db->temp_store Location of temporary database -** ----------------- -------------- ------------------------------ -** 0 any file (return 0) -** 1 1 file (return 0) -** 1 2 memory (return 1) -** 1 0 file (return 0) -** 2 1 file (return 0) -** 2 2 memory (return 1) -** 2 0 memory (return 1) -** 3 any memory (return 1) -*/ -SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ -#if SQLITE_TEMP_STORE==1 - return ( db->temp_store==2 ); -#endif -#if SQLITE_TEMP_STORE==2 - return ( db->temp_store!=1 ); -#endif -#if SQLITE_TEMP_STORE==3 - return 1; -#endif -#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 - return 0; -#endif + return SQLITE_OK; } /* -** Return UTF-8 encoded English language explanation of the most recent -** error. +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. */ -SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ - const char *z; - if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); - }else{ - testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); - assert( !db->mallocFailed ); - if( z==0 ){ - z = sqlite3ErrStr(db->errCode); + + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; } - sqlite3_mutex_leave(db->mutex); - return z; + + pMod->xClose(pC); + return rc; } -#ifndef SQLITE_OMIT_UTF16 + /* -** Return UTF-16 encoded English language explanation of the most recent -** error. +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. */ -SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ - static const u16 outOfMem[] = { - 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 - }; - static const u16 misuse[] = { - 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', - 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', - 'c', 'a', 'l', 'l', 'e', 'd', ' ', - 'o', 'u', 't', ' ', - 'o', 'f', ' ', - 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 - }; +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; - const void *z; - if( !db ){ - return (void *)outOfMem; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return (void *)misuse; - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = (void *)outOfMem; - }else{ - z = sqlite3_value_text16(db->pErr); - if( z==0 ){ - sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); - z = sqlite3_value_text16(db->pErr); - } - /* A malloc() may have failed within the call to sqlite3_value_text16() - ** above. If this is the case, then the db->mallocFailed flag needs to - ** be cleared before returning. Do this directly, instead of via - ** sqlite3ApiExit(), to avoid setting the database handle error message. - */ - db->mallocFailed = 0; + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; } - sqlite3_mutex_leave(db->mutex); - return z; + + *ppCollist = pEnd; + return nEntry; } -#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the most recent error code generated by an SQLite routine. If NULL is -** passed to this function, we assume a malloc() failed during sqlite3_open(). +** This function gathers 'y' or 'b' data for a single phrase. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; +static void fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ +){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + + while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); } - return db->errCode & db->errMask; } -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static void fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + fts3ExprLHitGather(pExpr->pLeft, p); + fts3ExprLHitGather(pExpr->pRight, p); + }else{ + fts3ExprLHits(pExpr, p); + } } - return db->errCode; } /* -** Return a string that describes the kind of error specified in the -** argument. For now, this simply calls the internal sqlite3ErrStr() -** function. +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* -** Invalidate all cached KeyInfo objects for database connection "db" +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. */ -static void invalidateCachedKeyInfo(sqlite3 *db){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - Index *pIdx; /* Each index */ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - if( pDb->pBt==0 ) continue; - sqlite3BtreeEnter(pDb->pBt); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; - } - } + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } - sqlite3BtreeLeave(pDb->pBt); } + + return rc; } -/* -** Create a new collating function for database "db". The name is zName -** and the encoding is enc. -*/ -static int createCollation( - sqlite3* db, - const char *zName, - u8 enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr ){ - CollSeq *pColl; - int enc2; - - assert( sqlite3_mutex_held(db->mutex) ); - - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - */ - enc2 = enc; - testcase( enc2==SQLITE_UTF16 ); - testcase( enc2==SQLITE_UTF16_ALIGNED ); - if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ - enc2 = SQLITE_UTF16NATIVE; + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; } - if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE_BKPT; + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + int nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; } - /* Check if this call is removing or replacing an existing collation - ** sequence. If so, and there are active VMs, return busy. If there - ** are no active VMs, invalidate any pre-compiled statements. - */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); - if( pColl && pColl->xCmp ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify collation sequence due to active statements"); - return SQLITE_BUSY; - } - sqlite3ExpirePreparedStatements(db); - invalidateCachedKeyInfo(db); + return nVal; +} - /* If collation sequence pColl was created directly by a call to - ** sqlite3_create_collation, and not generated by synthCollSeq(), - ** then any copies made by synthCollSeq() need to be invalidated. - ** Also, collation destructor - CollSeq.xDel() - function may need - ** to be called. - */ - if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); - int j; - for(j=0; j<3; j++){ - CollSeq *p = &aColl[j]; - if( p->enc==pColl->enc ){ - if( p->xDel ){ - p->xDel(p->pUser); - } - p->xCmp = 0; - } - } - } +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen +){ + sqlite3_stmt *pStmt; + const char *a; + sqlite3_int64 nDoc; + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK); + a = sqlite3_column_blob(pStmt, 0); + a += sqlite3Fts3GetVarint(a, &nDoc); + if( nDoc==0 ) return FTS_CORRUPT_VTAB; + *pnDoc = (u32)nDoc; + + if( paLen ) *paLen = a; return SQLITE_OK; } - /* -** This array defines hard upper bounds on limit values. The -** initializer must be kept in sync with the SQLITE_LIMIT_* -** #defines in sqlite3.h. +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. */ -static const int aHardLimit[] = { - SQLITE_MAX_LENGTH, - SQLITE_MAX_SQL_LENGTH, - SQLITE_MAX_COLUMN, - SQLITE_MAX_EXPR_DEPTH, - SQLITE_MAX_COMPOUND_SELECT, - SQLITE_MAX_VDBE_OP, - SQLITE_MAX_FUNCTION_ARG, - SQLITE_MAX_ATTACHED, - SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ - SQLITE_MAX_TRIGGER_DEPTH, - SQLITE_MAX_WORKER_THREADS, +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ }; -/* -** Make sure the hard limits are set to reasonable values +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. */ -#if SQLITE_MAX_LENGTH<100 -# error SQLITE_MAX_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH<100 -# error SQLITE_MAX_SQL_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH -# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH -#endif -#if SQLITE_MAX_COMPOUND_SELECT<2 -# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 -#endif -#if SQLITE_MAX_VDBE_OP<40 -# error SQLITE_MAX_VDBE_OP must be at least 40 -#endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 -#endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 -# error SQLITE_MAX_ATTACHED must be between 0 and 125 -#endif -#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 -# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 -#endif -#if SQLITE_MAX_COLUMN>32767 -# error SQLITE_MAX_COLUMN must not exceed 32767 -#endif -#if SQLITE_MAX_TRIGGER_DEPTH<1 -# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 -#endif -#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 -# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 -#endif +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} /* -** Change the value of a limit. Report the old value. -** If an invalid limit index is supplied, report -1. -** Make no changes but still report the old value if the -** new limit is negative. -** -** A new lower limit does not shrink existing constructs. -** It merely prevents new constructs that exceed the limit -** from forming. +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. */ -SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ - int oldLimit; - - - /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME - ** there is a hard upper bound set at compile-time by a C preprocessor - ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to - ** "_MAX_".) - */ - assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); - assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); - assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); - assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); - assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); - assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== - SQLITE_MAX_LIKE_PATTERN_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); - assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); - +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; - if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ - return -1; - } - oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ /* IMP: R-52476-28732 */ - if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - } - db->aLimit[limitId] = newLimit; + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); } - return oldLimit; /* IMP: R-53341-35419 */ -} + pIter->pRead = pRead; + return rc; +} + /* -** This function is used to parse both URIs and non-URI filenames passed by the -** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database -** URIs specified as part of ATTACH statements. -** -** The first argument to this function is the name of the VFS to use (or -** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" -** query parameter. The second argument contains the URI (or non-URI filename) -** itself. When this function is called the *pFlags variable should contain -** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains -** "cache=xxx" or "mode=xxx" query parameters. +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. ** -** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to -** the VFS that should be used to open the database file. *pzFile is set to -** point to a buffer containing the name of the file to open. It is the -** responsibility of the caller to eventually call sqlite3_free() to release -** this buffer. +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. ** -** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error -** message. It is the responsibility of the caller to eventually release -** this buffer by calling sqlite3_free(). +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. */ -SQLITE_PRIVATE int sqlite3ParseUri( - const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ - const char *zUri, /* Nul-terminated URI to parse */ - unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ - char **pzFile, /* OUT: Filename component of URI */ - char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ -){ - int rc = SQLITE_OK; - unsigned int flags = *pFlags; - const char *zVfs = zDefaultVfs; - char *zFile; - char c; - int nUri = sqlite3Strlen30(zUri); - - assert( *pzErrMsg==0 ); +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; - if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ - ){ - char *zOpt; - int eState; /* Parser state when parsing URI */ - int iIn; /* Input character index */ - int iOut = 0; /* Output character index */ - int nByte = nUri+2; /* Bytes of space to allocate */ + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen - ** method that there may be extra parameters following the file-name. */ - flags |= SQLITE_OPEN_URI; + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } - for(iIn=0; iInnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ - iIn = 5; -#ifndef SQLITE_ALLOW_URI_AUTHORITY - /* Discard the scheme and authority segments of the URI. */ - if( zUri[5]=='/' && zUri[6]=='/' ){ - iIn = 7; - while( zUri[iIn] && zUri[iIn]!='/' ) iIn++; - if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){ - *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", - iIn-7, &zUri[7]); - rc = SQLITE_ERROR; - goto parse_uri_out; + for(i=0; inPhrase; i++){ + int rc; + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) return rc; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; } } -#endif - /* Copy the filename and any query parameters into the zFile buffer. - ** Decode %HH escape codes along the way. - ** - ** Within this loop, variable eState may be set to 0, 1 or 2, depending - ** on the parsing context. As follows: - ** - ** 0: Parsing file-name. - ** 1: Parsing name section of a name=value query parameter. - ** 2: Parsing value section of a name=value query parameter. - */ - eState = 0; - while( (c = zUri[iIn])!=0 && c!='#' ){ - iIn++; - if( c=='%' - && sqlite3Isxdigit(zUri[iIn]) - && sqlite3Isxdigit(zUri[iIn+1]) - ){ - int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); - octet += sqlite3HexToInt(zUri[iIn++]); + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ - assert( octet>=0 && octet<256 ); - if( octet==0 ){ - /* This branch is taken when "%00" appears within the URI. In this - ** case we ignore all text in the remainder of the path, name or - ** value currently being parsed. So ignore the current character - ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' - && (eState!=0 || c!='?') - && (eState!=1 || (c!='=' && c!='&')) - && (eState!=2 || c!='&') - ){ - iIn++; - } - continue; - } - c = octet; - }else if( eState==1 && (c=='&' || c=='=') ){ - if( zFile[iOut-1]==0 ){ - /* An empty option name. Ignore this option altogether. */ - while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; - continue; - } - if( c=='&' ){ - zFile[iOut++] = '\0'; + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; }else{ - eState = 2; + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; } - c = 0; - }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ - c = 0; - eState = 1; } - zFile[iOut++] = c; + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; } - if( eState==1 ) zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - /* Check if there were any options specified that should be interpreted - ** here. Options that are interpreted here include "vfs" and those that - ** correspond to flags that may be passed to the sqlite3_open_v2() - ** method. */ - zOpt = &zFile[sqlite3Strlen30(zFile)+1]; - while( zOpt[0] ){ - int nOpt = sqlite3Strlen30(zOpt); - char *zVal = &zOpt[nOpt+1]; - int nVal = sqlite3Strlen30(zVal); + pInfo->aMatchinfo[iCol] = nLcs; + } - if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ - zVfs = zVal; - }else{ - struct OpenMode { - const char *z; - int mode; - } *aMode = 0; - char *zModeType = 0; - int mask = 0; - int limit = 0; + sqlite3_free(aIter); + return SQLITE_OK; +} - if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ - static struct OpenMode aCacheMode[] = { - { "shared", SQLITE_OPEN_SHAREDCACHE }, - { "private", SQLITE_OPEN_PRIVATECACHE }, - { 0, 0 } - }; +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; - mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; - aMode = aCacheMode; - limit = mask; - zModeType = "cache"; - } - if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ - static struct OpenMode aOpenMode[] = { - { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, - { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, - { "memory", SQLITE_OPEN_MEMORY }, - { 0, 0 } - }; + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; - mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; - aMode = aOpenMode; - limit = mask & flags; - zModeType = "access"; + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; } + break; - if( aMode ){ - int i; - int mode = 0; - for(i=0; aMode[i].z; i++){ - const char *z = aMode[i].z; - if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ - mode = aMode[i].mode; - break; + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; } } - if( mode==0 ){ - *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); - rc = SQLITE_ERROR; - goto parse_uri_out; - } - if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ - *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", - zModeType, zVal); - rc = SQLITE_PERM; - goto parse_uri_out; + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)nToken; } - flags = (flags & ~mask) | mode; } + sqlite3_reset(pSelectDocsize); + break; } - zOpt = &zVal[nVal+1]; + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } } - }else{ - zFile = sqlite3_malloc(nUri+2); - if( !zFile ) return SQLITE_NOMEM; - memcpy(zFile, zUri, nUri); - zFile[nUri] = '\0'; - zFile[nUri+1] = '\0'; - flags &= ~SQLITE_OPEN_URI; + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); } - *ppVfs = sqlite3_vfs_find(zVfs); - if( *ppVfs==0 ){ - *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); - rc = SQLITE_ERROR; - } - parse_uri_out: - if( rc!=SQLITE_OK ){ - sqlite3_free(zFile); - zFile = 0; - } - *pFlags = flags; - *pzFile = zFile; + sqlite3_reset(pSelect); return rc; } /* -** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" -** is UTF-8 encoded. +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int openDatabase( - const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned int flags, /* Operational flags */ - const char *zVfs /* Name of the VFS to use */ +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ ){ - sqlite3 *db; /* Store allocated handle here */ - int rc; /* Return code */ - int isThreadsafe; /* True for threadsafe connections */ - char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ - char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; - /* Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: - ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE - */ - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ){ - return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ - } + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; - if( sqlite3GlobalConfig.bCoreMutex==0 ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_NOMUTEX ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_FULLMUTEX ){ - isThreadsafe = 1; - }else{ - isThreadsafe = sqlite3GlobalConfig.bFullMutex; - } - if( flags & SQLITE_OPEN_PRIVATECACHE ){ - flags &= ~SQLITE_OPEN_SHAREDCACHE; - }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ - flags |= SQLITE_OPEN_SHAREDCACHE; + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; } - /* Remove harmful bits from the flags parameter - ** - ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were - ** dealt with in the previous code block. Besides these, the only - ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, - ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, - ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask - ** off all other flags. + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. */ - flags &= ~( SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | - SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX | - SQLITE_OPEN_WAL - ); - - /* Allocate the sqlite data structure */ - db = sqlite3MallocZero( sizeof(sqlite3) ); - if( db==0 ) goto opendb_out; - if( isThreadsafe ){ - db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( db->mutex==0 ){ - sqlite3_free(db); - db = 0; - goto opendb_out; - } - } - sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; - db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; - db->aDb = db->aDbStatic; + if( pCsr->pMIBuffer==0 ){ + int nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ - assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); - memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; - db->autoCommit = 1; - db->nextAutovac = -1; - db->szMmap = sqlite3GlobalConfig.szMmap; - db->nextPagesize = 0; - db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill -#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX - | SQLITE_AutoIndex -#endif -#if SQLITE_DEFAULT_FILE_FORMAT<4 - | SQLITE_LegacyFileFmt -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - | SQLITE_LoadExtension -#endif -#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS - | SQLITE_RecTriggers -#endif -#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS - | SQLITE_ForeignKeys -#endif - ; - sqlite3HashInit(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule); -#endif + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 - ** and UTF-16, so add a version for each to avoid any unnecessary - ** conversions. The only error that can occur here is a malloc() failure. - */ - createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); - if( db->mallocFailed ){ - goto opendb_out; - } - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); - assert( db->pDfltColl!=0 ); + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; - /* Parse the filename/URI argument. */ - db->openFlags = flags; - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); - sqlite3_free(zErrMsg); - goto opendb_out; + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; } - /* Open the backend database driver */ - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, - flags | SQLITE_OPEN_MAIN_DB); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_NOMEM ){ + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ rc = SQLITE_NOMEM; } - sqlite3Error(db, rc); - goto opendb_out; } - db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. - */ - db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; - db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } - db->magic = SQLITE_MAGIC_OPEN; - if( db->mallocFailed ){ - goto opendb_out; + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); } +} - /* Register all built-in functions, but do not attempt to read the - ** database schema yet. This is delayed until the first time the database - ** is accessed. - */ - sqlite3Error(db, SQLITE_OK); - sqlite3RegisterBuiltinFunctions(db); +/* +** Implementation of snippet() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. */ - rc = sqlite3_errcode(db); - if( rc==SQLITE_OK ){ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } - } + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ -#ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; } -#endif -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); - } -#endif + for(nSnippet=1; 1; nSnippet++){ -#ifdef SQLITE_ENABLE_FTS3 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); - } -#endif + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ -#ifdef SQLITE_ENABLE_ICU - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); - } -#endif + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif + for(iSnip=0; iSnipdfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); -#endif + memset(pFragment, 0, sizeof(*pFragment)); - if( rc ) sqlite3Error(db, rc); + /* Loop through all columns of the table being considered for snippets. + ** If the iCol argument to this function was negative, this means all + ** columns of the FTS3 table. Otherwise, only column iCol is considered. + */ + for(iRead=0; iReadnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + mCovered |= pFragment->covered; + } -opendb_out: - sqlite3_free(zOpen); - if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); - sqlite3_mutex_leave(db->mutex); - } - rc = sqlite3_errcode(db); - assert( db!=0 || rc==SQLITE_NOMEM ); - if( rc==SQLITE_NOMEM ){ - sqlite3_close(db); - db = 0; - }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; - } - *ppDb = db; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Opening a db handle. Fourth parameter is passed 0. */ - void *pArg = sqlite3GlobalConfig.pSqllogArg; - sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; } -#endif - return sqlite3ApiExit(0, rc); -} -/* -** Open a new database handle. -*/ -SQLITE_API int sqlite3_open( - const char *zFilename, - sqlite3 **ppDb -){ - return openDatabase(zFilename, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); -} -SQLITE_API int sqlite3_open_v2( - const char *filename, /* Database filename (UTF-8) */ - sqlite3 **ppDb, /* OUT: SQLite db handle */ - int flags, /* Flags */ - const char *zVfs /* Name of VFS module to use */ -){ - return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); -} + assert( nFToken>0 ); -#ifndef SQLITE_OMIT_UTF16 -/* -** Open a new database handle. -*/ -SQLITE_API int sqlite3_open16( - const void *zFilename, - sqlite3 **ppDb -){ - char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ - sqlite3_value *pVal; - int rc; + for(i=0; imutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} + +typedef struct TermOffset TermOffset; +typedef struct TermOffsetCtx TermOffsetCtx; + +struct TermOffset { + char *pList; /* Position-list */ + int iPos; /* Position just read from pList */ + int iOff; /* Offset of this term from read positions */ +}; + +struct TermOffsetCtx { + Fts3Cursor *pCsr; + int iCol; /* Column of table to populate aTerm for */ + int iTerm; + sqlite3_int64 iDocid; + TermOffset *aTerm; +}; /* -** Register a new collation sequence with the database handle db. +** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). */ -SQLITE_API int sqlite3_create_collation_v2( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ +static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx *)ctx; + int nTerm; /* Number of tokens in phrase */ + int iTerm; /* For looping through nTerm phrase terms */ + char *pList; /* Pointer to position list for phrase */ + int iPos = 0; /* First position in position-list */ int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#ifndef SQLITE_OMIT_UTF16 -/* -** Register a new collation sequence with the database handle db. -*/ -SQLITE_API int sqlite3_create_collation16( - sqlite3* db, - const void *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc = SQLITE_OK; - char *zName8; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); - if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); - sqlite3DbFree(db, zName8); + UNUSED_PARAMETER(iPhrase); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert( iPos>=0 ); } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** Implementation of offsets() function. */ -SQLITE_API int sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +SQLITE_PRIVATE void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ ){ - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = xCollNeeded; - db->xCollNeeded16 = 0; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ -#ifndef SQLITE_OMIT_UTF16 -/* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. -*/ -SQLITE_API int sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) -){ - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = 0; - db->xCollNeeded16 = xCollNeeded16; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_UTF16 */ + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } -#ifndef SQLITE_OMIT_DEPRECATED -/* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. -*/ -SQLITE_API int sqlite3_global_recover(void){ - return SQLITE_OK; -} -#endif + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); -/* -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. -*/ -SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ - return db->autoCommit; -} + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; -/* -** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They serve two purposes: -** -** 1. Serve as a convenient place to set a breakpoint in a debugger -** to detect when version error conditions occurs. -** -** 2. Invoke sqlite3_log() to provide the source code location where -** a low-level error is first detected. -*/ -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; -} -SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; -} -SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; -} + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; -#ifndef SQLITE_OMIT_DEPRECATED -/* -** This is a convenience routine that makes sure that all thread-specific -** data for this thread has been deallocated. -** -** SQLite no longer uses thread-specific data so this routine is now a -** no-op. It is retained for historical compatibility. -*/ -SQLITE_API void sqlite3_thread_cleanup(void){ -} -#endif + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); -/* -** Return meta information about a specific column of a database table. -** See comment in sqlite3.h (sqlite.h.in) for details. -*/ -#ifdef SQLITE_ENABLE_COLUMN_METADATA -SQLITE_API int sqlite3_table_column_metadata( - sqlite3 *db, /* Connection handle */ - const char *zDbName, /* Database name or NULL */ - const char *zTableName, /* Table name */ - const char *zColumnName, /* Column name */ - char const **pzDataType, /* OUTPUT: Declared data type */ - char const **pzCollSeq, /* OUTPUT: Collation sequence name */ - int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ - int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if column is auto-increment */ -){ - int rc; - char *zErrMsg = 0; - Table *pTab = 0; - Column *pCol = 0; - int iCol; + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } - char const *zDataType = 0; - char const *zCollSeq = 0; - int notnull = 0; - int primarykey = 0; - int autoinc = 0; + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; - /* Ensure the database schema has been loaded */ - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrMsg); - if( SQLITE_OK!=rc ){ - goto error_out; - } + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ - /* Locate the table in question */ - pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ - pTab = 0; - goto error_out; - } + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } - /* Find the column for which info is requested */ - if( sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - if( iCol>=0 ){ - pCol = &pTab->aCol[iCol]; - } - }else{ - for(iCol=0; iColnCol; iCol++){ - pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ - break; + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } } } - if( iCol==pTab->nCol ){ - pTab = 0; - goto error_out; + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; } - /* The following block stores the meta information that will be returned - ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey - ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. - ** - ** 2. The table is not a view and the column name identified an - ** explicitly declared column. Copy meta information from *pCol. - */ - if( pCol ){ - zDataType = pCol->zType; - zCollSeq = pCol->zColl; - notnull = pCol->notNull!=0; - primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; - autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); }else{ - zDataType = "INTEGER"; - primarykey = 1; - } - if( !zCollSeq ){ - zCollSeq = "BINARY"; + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); } + return; +} -error_out: - sqlite3BtreeLeaveAll(db); +/* +** Implementation of matchinfo() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; - /* Whether the function call succeeded or failed, set the output parameters - ** to whatever their local counterparts contain. If an error did occur, - ** this has the effect of zeroing all output parameters. - */ - if( pzDataType ) *pzDataType = zDataType; - if( pzCollSeq ) *pzCollSeq = zCollSeq; - if( pNotNull ) *pNotNull = notnull; - if( pPrimaryKey ) *pPrimaryKey = primarykey; - if( pAutoinc ) *pAutoinc = autoinc; + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } - if( SQLITE_OK==rc && !pTab ){ - sqlite3DbFree(db, zErrMsg); - zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, - zColumnName); - rc = SQLITE_ERROR; + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + }else{ + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); } - sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqlite3DbFree(db, zErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; } + #endif +/************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ /* -** Sleep for a little while. Return the amount of time slept. +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. */ -SQLITE_API int sqlite3_sleep(int ms){ - sqlite3_vfs *pVfs; - int rc; - pVfs = sqlite3_vfs_find(0); - if( pVfs==0 ) return 0; - /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. - */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); - return rc; -} +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ /* -** Enable or disable the extended result codes. +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. */ -SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ - sqlite3_mutex_enter(db->mutex); - db->errMask = onoff ? 0xffffffff : 0xff; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ } +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int bRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + /* -** Invoke the xFileControl method on a particular database. +** Destroy a tokenizer allocated by unicodeCreate(). */ -SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ - int rc = SQLITE_ERROR; - Btree *pBtree; - - sqlite3_mutex_enter(db->mutex); - pBtree = sqlite3DbNameToBtree(db, zDbName); - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - }else{ - rc = SQLITE_NOTFOUND; - } - sqlite3BtreeLeave(pBtree); +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); } - sqlite3_mutex_leave(db->mutex); - return rc; + return SQLITE_OK; } /* -** Interface to the testing logic. +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. */ -SQLITE_API int sqlite3_test_control(int op, ...){ - int rc = 0; -#ifndef SQLITE_OMIT_BUILTIN_TEST - va_list ap; - va_start(ap, op); - switch( op ){ - - /* - ** Save the current state of the PRNG. - */ - case SQLITE_TESTCTRL_PRNG_SAVE: { - sqlite3PrngSaveState(); - break; - } - - /* - ** Restore the state of the PRNG to the last state saved using - ** PRNG_SAVE. If PRNG_SAVE has never before been called, then - ** this verb acts like PRNG_RESET. - */ - case SQLITE_TESTCTRL_PRNG_RESTORE: { - sqlite3PrngRestoreState(); - break; - } +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + int iCode; + int nEntry = 0; - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. - */ - case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3_randomness(0,0); - break; - } + assert( bAlnum==0 || bAlnum==1 ); - /* - ** sqlite3_test_control(BITVEC_TEST, size, program) - ** - ** Run a test against a Bitvec object of size. The program argument - ** is an array of integers that defines the test. Return -1 on a - ** memory allocation error, 0 on success, or non-zero for an error. - ** See the sqlite3BitvecBuiltinTest() for additional information. - */ - case SQLITE_TESTCTRL_BITVEC_TEST: { - int sz = va_arg(ap, int); - int *aProg = va_arg(ap, int*); - rc = sqlite3BitvecBuiltinTest(sz, aProg); - break; + while( zaiException, (p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) - ** - ** Set the PENDING byte to the value in the argument, if X>0. - ** Make no changes if X==0. Return the value of the pending byte - ** as it existing before this routine was called. - ** - ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in - ** an incompatible database file format. Changing the PENDING byte - ** while any database connection is open results in undefined and - ** deleterious behavior. - */ - case SQLITE_TESTCTRL_PENDING_BYTE: { - rc = PENDING_BYTE; -#ifndef SQLITE_OMIT_WSD - { - unsigned int newVal = va_arg(ap, unsigned int); - if( newVal ) sqlite3PendingByte = newVal; + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = iCode; + nNew++; } -#endif - break; } + p->aiException = aNew; + p->nException = nNew; + } - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) - ** - ** This action provides a run-time test to see whether or not - ** assert() was enabled at compile-time. If X is true and assert() - ** is enabled, then the return value is true. If X is true and - ** assert() is disabled, then the return value is zero. If X is - ** false and assert() is enabled, then the assertion fires and the - ** process aborts. If X is false and assert() is disabled, then the - ** return value is zero. - */ - case SQLITE_TESTCTRL_ASSERT: { - volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); - rc = x; - break; - } + return SQLITE_OK; +} +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) - ** - ** This action provides a run-time test to see how the ALWAYS and - ** NEVER macros were defined at compile-time. - ** - ** The return value is ALWAYS(X). - ** - ** The recommended test is X==2. If the return value is 2, that means - ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the - ** default setting. If the return value is 1, then ALWAYS() is either - ** hard-coded to true or else it asserts if its argument is false. - ** The first behavior (hard-coded to true) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second - ** behavior (assert if the argument to ALWAYS() is false) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. - ** - ** The run-time test procedure might look something like this: - ** - ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ - ** // ALWAYS() and NEVER() are no-op pass-through macros - ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ - ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. - ** }else{ - ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. - ** } - */ - case SQLITE_TESTCTRL_ALWAYS: { - int x = va_arg(ap,int); - rc = ALWAYS(x); - break; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } } + } - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); - ** - ** The integer returned reveals the byte-order of the computer on which - ** SQLite is running: - ** - ** 1 big-endian, determined at run-time - ** 10 little-endian, determined at run-time - ** 432101 big-endian, determined at compile-time - ** 123410 little-endian, determined at compile-time - */ - case SQLITE_TESTCTRL_BYTEORDER: { - rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; - break; - } + return 0; +} - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) - ** - ** Enable or disable various optimizations for testing purposes. The - ** argument N is a bitmask of optimizations to be disabled. For normal - ** operation N should be 0. The idea is that a test program (like the - ** SQL Logic Test or SLT test module) can run the same SQL multiple times - ** with various optimizations disabled to verify that the same answer - ** is obtained in every case. - */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); - break; - } +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; -#ifdef SQLITE_N_KEYWORD - /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) - ** - ** If zWord is a keyword recognized by the parser, then return the - ** number of keywords. Or if zWord is not a keyword, return 0. - ** - ** This test feature is only available in the amalgamation since - ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite - ** is built using separate source files. - */ - case SQLITE_TESTCTRL_ISKEYWORD: { - const char *zWord = va_arg(ap, const char*); - int n = sqlite3Strlen30(zWord); - rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; - break; - } -#endif + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->bRemoveDiacritic = 1; - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); - break; - } + for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; } - - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); - ** - ** Set or clear a flag that indicates that the database file is always well- - ** formed and never corrupt. This flag is clear by default, indicating that - ** database files might have arbitrary corruption. Setting the flag during - ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. - */ - case SQLITE_TESTCTRL_NEVER_CORRUPT: { - sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); - break; + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->bRemoveDiacritic = 0; } - - - /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); - ** - ** Set the VDBE coverage callback function to xCallback with context - ** pointer ptr. - */ - case SQLITE_TESTCTRL_VDBE_COVERAGE: { -#ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); - sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); - sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); -#endif - break; + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); } - - /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ - case SQLITE_TESTCTRL_SORTER_MMAP: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->nMaxSorterMmap = va_arg(ap, int); - break; + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); } - - /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); - ** - ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if - ** not. - */ - case SQLITE_TESTCTRL_ISINIT: { - if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; - break; + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; } } - va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; return rc; } /* -** This is a utility routine, useful to VFS implementations, that checks -** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. -** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; - zFilename += sqlite3Strlen30(zFilename) + 1; - while( zFilename[0] ){ - int x = strcmp(zFilename, zParam); - zFilename += sqlite3Strlen30(zFilename) + 1; - if( x==0 ) return zFilename; - zFilename += sqlite3Strlen30(zFilename) + 1; +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; } - return 0; + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; } /* -** Return a boolean value for a query parameter. +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. */ -SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - bDflt = bDflt!=0; - return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** Return a 64-bit integer value for a query parameter. +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). */ -SQLITE_API sqlite3_int64 sqlite3_uri_int64( - const char *zFilename, /* Filename as passed to xOpen */ - const char *zParam, /* URI parameter sought */ - sqlite3_int64 bDflt /* return if parameter is missing */ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ ){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - sqlite3_int64 v; - if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ - bDflt = v; + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; -/* -** Return the Btree pointer identified by zDbName. Return NULL if not found. -*/ -SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int i; - for(i=0; inDb; i++){ - if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) - ){ - return db->aDb[i].pBt; + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; } - } - return 0; -} -/* -** Return the filename of the database associated with a database -** connection. -*/ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeGetFilename(pBt) : 0; + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, iCode) + || sqlite3FtsUnicodeIsdiacritic(iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; } /* -** Return 1 if database is read-only or 0 if read/write. Return -1 if -** no such database exists. +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. */ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; } -/************** End of main.c ************************************************/ -/************** Begin file notify.c ******************************************/ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ /* -** 2009 March 3 +** 2012 May 25 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -128438,623 +153679,366 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -************************************************************************* -** -** This file contains the implementation of the sqlite3_unlock_notify() -** API method and its associated functionality. +****************************************************************************** */ -/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - /* -** Public interfaces: -** -** sqlite3ConnectionBlocked() -** sqlite3ConnectionUnlocked() -** sqlite3ConnectionClosed() -** sqlite3_unlock_notify() +** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -#define assertMutexHeld() \ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) -/* -** Head of a linked list of all sqlite3 objects created by this process -** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection -** is not NULL. This variable may only accessed while the STATIC_MASTER -** mutex is held. -*/ -static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; +/* #include */ -#ifndef NDEBUG /* -** This function is a complex assert() that verifies the following -** properties of the blocked connections list: -** -** 1) Each entry in the list has a non-NULL value for either -** pUnlockConnection or pBlockingConnection, or both. -** -** 2) All entries in the list that share a common value for -** xUnlockNotify are grouped together. +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. ** -** 3) If the argument db is not NULL, then none of the entries in the -** blocked connections list have pUnlockConnection or pBlockingConnection -** set to db. This is used when closing connection db. +** The results are undefined if the value passed to this function +** is less than zero. */ -static void checkListProperties(sqlite3 *db){ - sqlite3 *p; - for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ - int seen = 0; - sqlite3 *p2; - - /* Verify property (1) */ - assert( p->pUnlockConnection || p->pBlockingConnection ); - - /* Verify property (2) */ - for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ - if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; - assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); - assert( db==0 || p->pUnlockConnection!=db ); - assert( db==0 || p->pBlockingConnection!=db ); - } - } -} -#else -# define checkListProperties(x) -#endif +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; -/* -** Remove connection db from the blocked connections list. If connection -** db is not currently a part of the list, this function is a no-op. -*/ -static void removeFromBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ - if( *pp==db ){ - *pp = (*pp)->pNextBlocked; - break; + if( c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); } + return 1; } -/* -** Add connection db to the blocked connections list. It is assumed -** that it is not already a part of the list. -*/ -static void addToBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for( - pp=&sqlite3BlockedList; - *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; - pp=&(*pp)->pNextBlocked - ); - db->pNextBlocked = *pp; - *pp = db; -} - -/* -** Obtain the STATIC_MASTER mutex. -*/ -static void enterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - checkListProperties(0); -} - -/* -** Release the STATIC_MASTER mutex. -*/ -static void leaveMutex(void){ - assertMutexHeld(); - checkListProperties(0); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); -} /* -** Register an unlock-notify callback. -** -** This is called after connection "db" has attempted some operation -** but has received an SQLITE_LOCKED error because another connection -** (call it pOther) in the same process was busy using the same shared -** cache. pOther is found by looking at db->pBlockingConnection. -** -** If there is no blocking connection, the callback is invoked immediately, -** before this routine returns. -** -** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate -** a deadlock. -** -** Otherwise, make arrangements to invoke xNotify when pOther drops -** its locks. -** -** Each call to this routine overrides any prior callbacks registered -** on the same "db". If xNotify==0 then any prior callbacks are immediately -** cancelled. +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. */ -SQLITE_API int sqlite3_unlock_notify( - sqlite3 *db, - void (*xNotify)(void **, int), - void *pArg -){ - int rc = SQLITE_OK; - - sqlite3_mutex_enter(db->mutex); - enterMutex(); - - if( xNotify==0 ){ - removeFromBlockedList(db); - db->pBlockingConnection = 0; - db->pUnlockConnection = 0; - db->xUnlockNotify = 0; - db->pUnlockArg = 0; - }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a - ** blocking transaction. In either case, invoke the notify callback - ** immediately. - */ - xNotify(&pArg, 1); - }else{ - sqlite3 *p; +static int remove_diacritic(int c){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, + 62924, 63050, 63082, 63274, 63390, + }; + char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', + 'e', 'i', 'o', 'u', 'y', + }; - for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} - if( p ){ - rc = SQLITE_LOCKED; /* Deadlock detected. */ + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; }else{ - db->pUnlockConnection = db->pBlockingConnection; - db->xUnlockNotify = xNotify; - db->pUnlockArg = pArg; - removeFromBlockedList(db); - addToBlockedList(db); + iHi = iTest-1; } } - - leaveMutex(); - assert( !db->mallocFailed ); - sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); - sqlite3_mutex_leave(db->mutex); - return rc; + assert( key>=aDia[iRes] ); + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); } + /* -** This function is called while stepping or preparing a statement -** associated with connection db. The operation will return SQLITE_LOCKED -** to the user because it requires a lock that will not be available -** until connection pBlocker concludes its current transaction. +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. */ -SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ - enterMutex(); - if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ - addToBlockedList(db); - } - db->pBlockingConnection = pBlocker; - leaveMutex(); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & (1 << (c-768))) : + (mask1 & (1 << (c-768-32))); } + /* -** This function is called when -** the transaction opened by database db has just finished. Locks held -** by database connection db have been released. -** -** This function loops through each entry in the blocked connections -** list and does the following: -** -** 1) If the sqlite3.pBlockingConnection member of a list entry is -** set to db, then set pBlockingConnection=0. -** -** 2) If the sqlite3.pUnlockConnection member of a list entry is -** set to db, then invoke the configured unlock-notify callback and -** set pUnlockConnection=0. +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. ** -** 3) If the two steps above mean that pBlockingConnection==0 and -** pUnlockConnection==0, remove the entry from the blocked connections -** list. +** The results are undefined if the value passed to this function +** is less than zero. */ -SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ - void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ - int nArg = 0; /* Number of entries in aArg[] */ - sqlite3 **pp; /* Iterator variable */ - void **aArg; /* Arguments to the unlock callback */ - void **aDyn = 0; /* Dynamically allocated space for aArg[] */ - void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; - aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ + int ret = c; - /* This loop runs once for each entry in the blocked-connections list. */ - for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ - sqlite3 *p = *pp; + assert( c>=0 ); + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); - /* Step 1. */ - if( p->pBlockingConnection==db ){ - p->pBlockingConnection = 0; - } + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; - /* Step 2. */ - if( p->pUnlockConnection==db ){ - assert( p->xUnlockNotify ); - if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ - xUnlockNotify(aArg, nArg); - nArg = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; } + } + assert( iRes<0 || c>=aEntry[iRes].iCode ); - sqlite3BeginBenignMalloc(); - assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); - assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); - if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) - ){ - /* The aArg[] array needs to grow. */ - void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); - if( pNew ){ - memcpy(pNew, aArg, nArg*sizeof(void *)); - sqlite3_free(aDyn); - aDyn = aArg = pNew; - }else{ - /* This occurs when the array of context pointers that need to - ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to - ** allocate a larger array from the heap has failed. - ** - ** This is a difficult situation to handle. Returning an error - ** code to the caller is insufficient, as even if an error code - ** is returned the transaction on connection db will still be - ** closed and the unlock-notify callbacks on blocked connections - ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never - ** arrive. - ** - ** Instead, invoke the unlock-notify callback with the context - ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without - ** requiring any dynamic allocation. This is sub-optimal because - ** it means that instead of one callback with a large array of - ** context pointers the application will receive two or more - ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple - ** connections. But it is the best that can be done under the - ** circumstances. - */ - xUnlockNotify(aArg, nArg); - nArg = 0; - } + if( iRes>=0 ){ + const struct TableEntry *p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } - sqlite3EndBenignMalloc(); - - aArg[nArg++] = p->pUnlockArg; - xUnlockNotify = p->xUnlockNotify; - p->pUnlockConnection = 0; - p->xUnlockNotify = 0; - p->pUnlockArg = 0; } - /* Step 3. */ - if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ - /* Remove connection p from the blocked connections list. */ - *pp = p->pNextBlocked; - p->pNextBlocked = 0; - }else{ - pp = &p->pNextBlocked; - } + if( bRemoveDiacritic ) ret = remove_diacritic(ret); } - - if( nArg!=0 ){ - xUnlockNotify(aArg, nArg); + + else if( c>=66560 && c<66600 ){ + ret = c + 40; } - sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ -} -/* -** This is called when the database connection passed as an argument is -** being closed. The connection is removed from the blocked list. -*/ -SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ - sqlite3ConnectionUnlocked(db); - enterMutex(); - removeFromBlockedList(db); - checkListProperties(db); - leaveMutex(); + return ret; } -#endif - -/************** End of notify.c **********************************************/ -/************** Begin file fts3.c ********************************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is an SQLite module implementing full-text search. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ - -/* The full-text index is stored in a series of b+tree (-like) -** structures called segments which map terms to doclists. The -** structures are like b+trees in layout, but are constructed from the -** bottom up in optimal fashion and are not updatable. Since trees -** are built from the bottom up, things will be described from the -** bottom up. -** -** -**** Varints **** -** The basic unit of encoding is a variable-length integer called a -** varint. We encode variable-length integers in little-endian order -** using seven bits * per byte as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** and so on. -** -** This is similar in concept to how sqlite encodes "varints" but -** the encoding is not the same. SQLite varints are big-endian -** are are limited to 9 bytes in length whereas FTS3 varints are -** little-endian and can be up to 10 bytes in length (in theory). -** -** Example encodings: -** -** 1: 0x01 -** 127: 0x7f -** 128: 0x81 0x00 -** -** -**** Document lists **** -** A doclist (document list) holds a docid-sorted list of hits for a -** given term. Doclists hold docids and associated token positions. -** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first -** word of the document has a position of 0. -** -** FTS3 used to optionally store character offsets using a compile-time -** option. But that functionality is no longer supported. -** -** A doclist is stored like this: -** -** array { -** varint docid; (delta from previous doclist) -** array { (position list for column 0) -** varint position; (2 more than the delta from previous position) -** } -** array { -** varint POS_COLUMN; (marks start of position list for new column) -** varint column; (index of new column) -** array { -** varint position; (2 more than the delta from previous position) -** } -** } -** varint POS_END; (marks end of positions for this document. -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals -** ending a position list array. POS_END is 0. POS_COLUMN is 1. -** The positions numbers are not stored literally but rather as two more -** than the difference from the prior position, or the just the position plus -** 2 for the first position. Example: -** -** label: A B C D E F G H I J K -** value: 123 5 9 1 1 14 35 0 234 72 0 -** -** The 123 value is the first docid. For column zero in this document -** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 -** at D signals the start of a new column; the 1 at E indicates that the -** new column is column number 1. There are two positions at 12 and 45 -** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the delta to next docid (357). It has one position 70 -** (72-2) and then terminates with the 0 at K. -** -** A "position-list" is the list of positions for multiple columns for -** a single docid. A "column-list" is the set of positions for a single -** column. Hence, a position-list consists of one or more column-lists, -** a document record consists of a docid followed by a position-list and -** a doclist consists of one or more document records. -** -** A bare doclist omits the position information, becoming an -** array of varint-encoded docids. -** -**** Segment leaf nodes **** -** Segment leaf nodes store terms and doclists, ordered by term. Leaf -** nodes are written using LeafWriter, and read using LeafReader (to -** iterate through a single leaf node's data) and LeavesReader (to -** iterate through a segment's entire leaf layer). Leaf nodes have -** the format: -** -** varint iHeight; (height from leaf level, always 0) -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of prefix shared with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix];(unshared suffix of next term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. -** -** Leaf nodes are broken into blocks which are stored contiguously in -** the %_segments table in sorted order. This means that when the end -** of a node is reached, the next term is in the node with the next -** greater node id. -** -** New data is spilled to a new leaf node when the current node -** exceeds LEAF_MAX bytes (default 2048). New data which itself is -** larger than STANDALONE_MIN (default 1024) is placed in a standalone -** node (a leaf node with a single term and doclist). The goal of -** these settings is to pack together groups of small doclists while -** making it efficient to directly access large doclists. The -** assumption is that large doclists represent terms which are more -** likely to be query targets. -** -** TODO(shess) It may be useful for blocking decisions to be more -** dynamic. For instance, it may make more sense to have a 2.5k leaf -** node rather than splitting into 2k and .5k nodes. My intuition is -** that this might extend through 2x or 4x the pagesize. -** -** -**** Segment interior nodes **** -** Segment interior nodes store blockids for subtree nodes and terms -** to describe what data is stored by the each subtree. Interior -** nodes are written using InteriorWriter, and read using -** InteriorReader. InteriorWriters are created as needed when -** SegmentWriter creates new leaf nodes, or when an interior node -** itself grows too big and must be split. The format of interior -** nodes: -** -** varint iHeight; (height from leaf level, always >0) -** varint iBlockid; (block id of node's leftmost subtree) -** optional { -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of shared prefix with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix]; (unshared suffix of next term) -** } -** } -** -** Here, optional { X } means an optional element, while array { X } -** means zero or more occurrences of X, adjacent in memory. -** -** An interior node encodes n terms separating n+1 subtrees. The -** subtree blocks are contiguous, so only the first subtree's blockid -** is encoded. The subtree at iBlockid will contain all terms less -** than the first term encoded (or all terms if no term is encoded). -** Otherwise, for terms greater than or equal to pTerm[i] but less -** than pTerm[i+1], the subtree for that term will be rooted at -** iBlockid+i. Interior nodes only store enough term data to -** distinguish adjacent children (if the rightmost term of the left -** child is "something", and the leftmost term of the right child is -** "wicked", only "w" is stored). -** -** New data is spilled to a new interior node at the same height when -** the current node exceeds INTERIOR_MAX bytes (default 2048). -** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing -** interior nodes and making the tree too skinny. The interior nodes -** at a given height are naturally tracked by interior nodes at -** height+1, and so on. -** -** -**** Segment directory **** -** The segment directory in table %_segdir stores meta-information for -** merging and deleting segments, and also the root node of the -** segment's tree. -** -** The root node is the top node of the segment's tree after encoding -** the entire segment, restricted to ROOT_MAX bytes (default 1024). -** This could be either a leaf node or an interior node. If the top -** node requires more than ROOT_MAX bytes, it is flushed to %_segments -** and a new root interior node is generated (which should always fit -** within ROOT_MAX because it only needs space for 2 varints, the -** height and the blockid of the previous root). -** -** The meta-information in the segment directory is: -** level - segment level (see below) -** idx - index within level -** - (level,idx uniquely identify a segment) -** start_block - first leaf node -** leaves_end_block - last leaf node -** end_block - last block (including interior nodes) -** root - contents of root node -** -** If the root node is a leaf node, then start_block, -** leaves_end_block, and end_block are all 0. -** -** -**** Segment merging **** -** To amortize update costs, segments are grouped into levels and -** merged in batches. Each increase in level represents exponentially -** more documents. -** -** New documents (actually, document updates) are tokenized and -** written individually (using LeafWriter) to a level 0 segment, with -** incrementing idx. When idx reaches MERGE_COUNT (default 16), all -** level 0 segments are merged into a single level 1 segment. Level 1 -** is populated like level 0, and eventually MERGE_COUNT level 1 -** segments are merged to a single level 2 segment (representing -** MERGE_COUNT^2 updates), and so on. -** -** A segment merge traverses all segments at a given level in -** parallel, performing a straightforward sorted merge. Since segment -** leaf nodes are written in to the %_segments table in order, this -** merge traverses the underlying sqlite disk structures efficiently. -** After the merge, all segment blocks from the merged level are -** deleted. -** -** MERGE_COUNT controls how often we merge segments. 16 seems to be -** somewhat of a sweet spot for insertion performance. 32 and 64 show -** very similar performance numbers to 16 on insertion, though they're -** a tiny bit slower (perhaps due to more overhead in merge-time -** sorting). 8 is about 20% slower than 16, 4 about 50% slower than -** 16, 2 about 66% slower than 16. -** -** At query time, high MERGE_COUNT increases the number of segments -** which need to be scanned and merged. For instance, with 100k docs -** inserted: -** -** MERGE_COUNT segments -** 16 25 -** 8 12 -** 4 10 -** 2 6 -** -** This appears to have only a moderate impact on queries for very -** frequent terms (which are somewhat dominated by segment merge -** costs), and infrequent and non-existent terms still seem to be fast -** even with many segments. -** -** TODO(shess) That said, it would be nice to have a better query-side -** argument for MERGE_COUNT of 16. Also, it is possible/likely that -** optimizations to things like doclist merging will swing the sweet -** spot around. -** -** -** -**** Handling of deletions and updates **** -** Since we're using a segmented structure, with no docid-oriented -** index into the term index, we clearly cannot simply update the term -** index when a document is deleted or updated. For deletions, we -** write an empty doclist (varint(docid) varint(POS_END)), for updates -** we simply write the new doclist. Segment merges overwrite older -** data for a particular docid with newer data, so deletes or updates -** will eventually overtake the earlier data and knock it out. The -** query logic likewise merges doclists so that newer data knocks out -** older data. -*/ +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ -/************** Include fts3Int.h in the middle of fts3.c ********************/ -/************** Begin file fts3Int.h *****************************************/ +/************** End of fts3_unicode2.c ***************************************/ +/************** Begin file rtree.c *******************************************/ /* -** 2009 Nov 12 +** 2001 September 15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -129063,8605 +154047,9512 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -****************************************************************************** -** -*/ -#ifndef _FTSINT_H -#define _FTSINT_H - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* -** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +************************************************************************* +** This file contains code for implementations of the r-tree and r*-tree +** algorithms packaged as an SQLite virtual table module. */ -#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 -#endif -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* If not building as part of the core, include sqlite3ext.h. */ -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - -/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ -/************** Begin file fts3_tokenizer.h **********************************/ /* -** 2006 July 10 +** Database Format of R-Tree Tables +** -------------------------------- ** -** The author disclaims copyright to this source code. +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. ** -************************************************************************* -** Defines the interface to tokenizers used by fulltext-search. There -** are three basic components: +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) ** -** sqlite3_tokenizer_module is a singleton defining the tokenizer -** interface functions. This is essentially the class structure for -** tokenizers. +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. ** -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps -** including customization information defined at creation time. +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: ** -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate -** tokens from a particular input. -*/ -#ifndef _FTS3_TOKENIZER_H_ -#define _FTS3_TOKENIZER_H_ - -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. -** If tokenizers are to be allowed to call sqlite3_*() functions, then -** we will need a way to register the API consistently. -*/ - -/* -** Structures used by the tokenizer interface. When a new tokenizer -** implementation is registered, the caller provides a pointer to -** an sqlite3_tokenizer_module containing pointers to the callback -** functions that make up an implementation. +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. ** -** When an fts3 table is created, it passes any arguments passed to -** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the -** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer -** implementation. The xCreate() function in turn returns an -** sqlite3_tokenizer structure representing the specific tokenizer to -** be used for the fts3 table (customized by the tokenizer clause arguments). +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. ** -** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() -** method is called. It returns an sqlite3_tokenizer_cursor object -** that may be used to tokenize a specific input buffer based on -** the tokenization rules supplied by a specific sqlite3_tokenizer -** object. +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. */ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; -struct sqlite3_tokenizer_module { +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) - /* - ** Structure version. Should always be set to 0 or 1. - */ - int iVersion; +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) - ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". - ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialized by - ** this callback. The caller will do so. - */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ - ); +/* #include */ +/* #include */ +/* #include */ - /* - ** Destroy an existing tokenizer. The fts3 module calls this method - ** exactly once for each successful call to xCreate(). - */ - int (*xDestroy)(sqlite3_tokenizer *pTokenizer); +#ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" +typedef sqlite3_int64 i64; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +#endif - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); +typedef struct Rtree Rtree; +typedef struct RtreeCursor RtreeCursor; +typedef struct RtreeNode RtreeNode; +typedef struct RtreeCell RtreeCell; +typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; +typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. - ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. - ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. - */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); +/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ +#define RTREE_MAX_DIMENSIONS 5 - /*********************************************************************** - ** Methods below this point are only available if iVersion>=1. - */ +/* Size of hash table Rtree.aHash. This hash table is not expected to +** ever contain very many entries, so a fixed number of buckets is +** used. +*/ +#define HASHSIZE 97 - /* - ** Configure the language id of a tokenizer cursor. - */ - int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); -}; +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ -}; +/* +** An rtree virtual-table object. +*/ +struct Rtree { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* Host database connection */ + int iNodeSize; /* Size in bytes of each node in the node table */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ + int iDepth; /* Current depth of the r-tree structure */ + char *zDb; /* Name of database containing r-tree table */ + char *zName; /* Name of r-tree table */ + int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ -struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ - /* Tokenizer implementations will typically add additional fields */ -}; + /* List of nodes removed during a CondenseTree operation. List is + ** linked together via the pointer normally used for hash chains - + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** headed by the node (leaf nodes have RtreeNode.iNode==0). + */ + RtreeNode *pDeleted; + int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ -int fts3_global_term_cnt(int iTerm, int iCol); -int fts3_term_cnt(int iTerm, int iCol); + /* Statements to read/write/delete a record from xxx_node */ + sqlite3_stmt *pReadNode; + sqlite3_stmt *pWriteNode; + sqlite3_stmt *pDeleteNode; + /* Statements to read/write/delete a record from xxx_rowid */ + sqlite3_stmt *pReadRowid; + sqlite3_stmt *pWriteRowid; + sqlite3_stmt *pDeleteRowid; -#endif /* _FTS3_TOKENIZER_H_ */ + /* Statements to read/write/delete a record from xxx_parent */ + sqlite3_stmt *pReadParent; + sqlite3_stmt *pWriteParent; + sqlite3_stmt *pDeleteParent; -/************** End of fts3_tokenizer.h **************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ -/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ -/************** Begin file fts3_hash.h ***************************************/ -/* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for the generic hash-table implementation -** used in SQLite. We've modified it slightly to serve as a standalone -** hash table implementation for the full-text indexing module. -** -*/ -#ifndef _FTS3_HASH_H_ -#define _FTS3_HASH_H_ + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ +}; -/* Forward declarations of structures. */ -typedef struct Fts3Hash Fts3Hash; -typedef struct Fts3HashElem Fts3HashElem; +/* Possible values for Rtree.eCoordType: */ +#define RTREE_COORD_REAL32 0 +#define RTREE_COORD_INT32 1 -/* A complete hash table is an instance of the following structure. -** The internals of this structure are intended to be opaque -- client -** code should not attempt to access or modify the fields of this structure -** directly. Change this structure only by using the routines below. -** However, many of the "procedures" and "functions" for modifying and -** accessing this structure are really macros, so we can't really make -** this structure opaque. +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. */ -struct Fts3Hash { - char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - Fts3HashElem *first; /* The first element of the array */ - int htsize; /* Number of buckets in the hash table */ - struct _fts3ht { /* the hash table */ - int count; /* Number of entries with this hash */ - Fts3HashElem *chain; /* Pointer to first entry with this hash */ - } *ht; -}; +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif -/* Each element in the hash table is an instance of the following -** structure. All elements are stored on a single doubly-linked list. +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. ** -** Again, this structure is intended to be opaque, but it can't really -** be opaque because it is used by macros. +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. */ -struct Fts3HashElem { - Fts3HashElem *next, *prev; /* Next and previous elements in the table */ - void *data; /* Data associated with this element */ - void *pKey; int nKey; /* Key associated with this element */ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ }; /* -** There are 2 different modes of operation for a hash table: -** -** FTS3_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is respected in comparisons. +** The minimum number of cells allowed for a node is a third of the +** maximum. In Gutman's notation: ** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. +** m = M/3 ** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +** If an R*-tree "Reinsert" operation is required, the same number of +** cells are removed from the overfull node and reinserted into the tree. */ -#define FTS3_HASH_STRING 1 -#define FTS3_HASH_BINARY 2 +#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) +#define RTREE_REINSERT(p) RTREE_MINCELLS(p) +#define RTREE_MAXCELLS 51 /* -** Access routines. To delete, insert a NULL pointer. +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. */ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); -SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); +#define RTREE_MAX_DEPTH 40 -/* -** Shorthand for the functions above -*/ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear -#define fts3HashFindElem sqlite3Fts3HashFindElem /* -** Macros for looping over all elements of a hash table. The idiom is -** like this: -** -** Fts3Hash h; -** Fts3HashElem *p; -** ... -** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ -** SomeStructure *pData = fts3HashData(p); -** // do something with pData -** } +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. */ -#define fts3HashFirst(H) ((H)->first) -#define fts3HashNext(E) ((E)->next) -#define fts3HashData(E) ((E)->data) -#define fts3HashKey(E) ((E)->pKey) -#define fts3HashKeysize(E) ((E)->nKey) +#define RTREE_CACHE_SZ 5 -/* -** Number of entries in a hash table +/* +** An rtree cursor object. */ -#define fts3HashCount(H) ((H)->count) - -#endif /* _FTS3_HASH_H_ */ +struct RtreeCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ + int iStrategy; /* Copy of idxNum search parameter */ + int nConstraint; /* Number of entries in aConstraint */ + RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ +}; -/************** End of fts3_hash.h *******************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) /* -** This constant determines the maximum depth of an FTS expression tree -** that the library will create and use. FTS uses recursion to perform -** various operations on the query tree, so the disadvantage of a large -** limit is that it may allow very large queries to use large amounts -** of stack space (perhaps causing a stack overflow). +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. */ -#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH -# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 -#endif - +union RtreeCoord { + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ +}; /* -** This constant controls how often segments are merged. Once there are -** FTS3_MERGE_COUNT segments of level N, they are merged into a single -** segment of level N+1. +** The argument is an RtreeCoord. Return the value stored within the RtreeCoord +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. */ -#define FTS3_MERGE_COUNT 16 +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif /* -** This is the maximum amount of data (in bytes) to store in the -** Fts3Table.pendingTerms hash table. Normally, the hash table is -** populated as documents are inserted/updated/deleted in a transaction -** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new -** segment is created and the hash table cleared immediately. +** A search constraint. */ -#define FTS3_MAX_PENDING_DATA (1*1024*1024) +struct RtreeConstraint { + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ +}; -/* -** Macro to return the number of elements in an array. SQLite has a -** similar macro called ArraySize(). Use a different name to avoid -** a collision when building an amalgamation with built-in FTS3. -*/ -#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) +/* Possible values for RtreeConstraint.op */ +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif -#ifndef MAX -# define MAX(x,y) ((x)>(y)?(x):(y)) -#endif +/* +** An rtree structure node. +*/ +struct RtreeNode { + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ +}; -/* -** Maximum length of a varint encoded integer. The varint format is different -** from that used by SQLite, so the maximum length is 10, not 9. +/* Return the number of cells in a node */ +#define NCELL(pNode) readInt16(&(pNode)->zData[2]) + +/* +** A single cell from a node, deserialized */ -#define FTS3_VARINT_MAX 10 +struct RtreeCell { + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + /* -** FTS4 virtual tables may maintain multiple indexes - one index of all terms -** in the document set and zero or more prefix indexes. All indexes are stored -** as one or more b+-trees in the %_segments and %_segdir tables. -** -** It is possible to determine which index a b+-tree belongs to based on the -** value stored in the "%_segdir.level" column. Given this value L, the index -** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with -** level values between 0 and 1023 (inclusive) belong to index 0, all levels -** between 1024 and 2047 to index 1, and so on. +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. ** -** It is considered impossible for an index to use more than 1024 levels. In -** theory though this may happen, but only after at least -** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). */ -#define FTS3_SEGDIR_MAXLEVEL 1024 -#define FTS3_SEGDIR_MAXLEVEL_STR "1024" +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; +}; -/* -** The testcase() macro is only used by the amalgamation. If undefined, -** make it a no-op. -*/ -#ifndef testcase -# define testcase(X) -#endif /* -** Terminator values for position-lists and column-lists. +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). */ -#define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ +#define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the -** amalgamation. -*/ -#ifndef SQLITE_AMALGAMATION -/* -** Macros indicating that conditional expressions are always true or -** false. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#else -# define ALWAYS(x) (x) -# define NEVER(x) (x) +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + sqlite3_value **apSqlParam; /* Original SQL parameter values */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ +}; + +#ifndef MAX +# define MAX(x,y) ((x) < (y) ? (y) : (x)) +#endif +#ifndef MIN +# define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif /* -** Internal types used by SQLite. +** Functions to deserialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The deserialized value is returned. */ -typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ -typedef short int i16; /* 2-byte (or larger) signed integer */ -typedef unsigned int u32; /* 4-byte unsigned integer */ -typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ -typedef sqlite3_int64 i64; /* 8-byte signed integer */ +static int readInt16(u8 *p){ + return (p[0]<<8) + p[1]; +} +static void readCoord(u8 *p, RtreeCoord *pCoord){ + pCoord->u = ( + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + + (((u32)p[3]) << 0) + ); +} +static i64 readInt64(u8 *p){ + return ( + (((i64)p[0]) << 56) + + (((i64)p[1]) << 48) + + (((i64)p[2]) << 40) + + (((i64)p[3]) << 32) + + (((i64)p[4]) << 24) + + (((i64)p[5]) << 16) + + (((i64)p[6]) << 8) + + (((i64)p[7]) << 0) + ); +} /* -** Macro used to suppress compiler warnings for unused parameters. +** Functions to serialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The value returned is the number of bytes written +** to the argument buffer (always 2, 4 and 8 respectively). */ -#define UNUSED_PARAMETER(x) (void)(x) +static int writeInt16(u8 *p, int i){ + p[0] = (i>> 8)&0xFF; + p[1] = (i>> 0)&0xFF; + return 2; +} +static int writeCoord(u8 *p, RtreeCoord *pCoord){ + u32 i; + assert( sizeof(RtreeCoord)==4 ); + assert( sizeof(u32)==4 ); + i = pCoord->u; + p[0] = (i>>24)&0xFF; + p[1] = (i>>16)&0xFF; + p[2] = (i>> 8)&0xFF; + p[3] = (i>> 0)&0xFF; + return 4; +} +static int writeInt64(u8 *p, i64 i){ + p[0] = (i>>56)&0xFF; + p[1] = (i>>48)&0xFF; + p[2] = (i>>40)&0xFF; + p[3] = (i>>32)&0xFF; + p[4] = (i>>24)&0xFF; + p[5] = (i>>16)&0xFF; + p[6] = (i>> 8)&0xFF; + p[7] = (i>> 0)&0xFF; + return 8; +} /* -** Activate assert() only if SQLITE_TEST is enabled. +** Increment the reference count of node p. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif +static void nodeReference(RtreeNode *p){ + if( p ){ + p->nRef++; + } +} /* -** The TESTONLY macro is used to enclose variable declarations or -** other bits of code that are needed to support the arguments -** within testcase() and assert() macros. +** Clear the content of node p (set all bytes to 0x00). */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -# define TESTONLY(X) X -#else -# define TESTONLY(X) -#endif - -#endif /* SQLITE_AMALGAMATION */ - -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); -# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() -#else -# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB -#endif - -typedef struct Fts3Table Fts3Table; -typedef struct Fts3Cursor Fts3Cursor; -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3PhraseToken Fts3PhraseToken; - -typedef struct Fts3Doclist Fts3Doclist; -typedef struct Fts3SegFilter Fts3SegFilter; -typedef struct Fts3DeferredToken Fts3DeferredToken; -typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3MultiSegReader Fts3MultiSegReader; +static void nodeZero(Rtree *pRtree, RtreeNode *p){ + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; +} /* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. +** Given a node number iNode, return the corresponding key to use +** in the Rtree.aHash table. */ -struct Fts3Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of named columns in virtual table */ - char **azColumn; /* column names. malloced */ - u8 *abNotindexed; /* True for 'notindexed' columns */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - char *zContentTbl; /* content=xxx option, or NULL */ - char *zLanguageid; /* languageid=xxx option, or NULL */ - int nAutoincrmerge; /* Value configured by 'automerge' */ - u32 nLeafAdd; /* Number of leaf blocks added this trans */ - - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. - */ - sqlite3_stmt *aStmt[40]; - - char *zReadExprlist; - char *zWriteExprlist; - - int nNodeSize; /* Soft limit for node size */ - u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists (2==unknown) */ - u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bDescIdx; /* True if doclists are in reverse order */ - u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ - int nPgsz; /* Page size for host database */ - char *zSegmentsTbl; /* Name of %_segments table */ - sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - - /* - ** The following array of hash tables is used to buffer pending index - ** updates during transactions. All pending updates buffered at any one - ** time must share a common language-id (see the FTS4 langid= feature). - ** The current language id is stored in variable iPrevLangid. - ** - ** A single FTS4 table may have multiple full-text indexes. For each index - ** there is an entry in the aIndex[] array. Index 0 is an index of all the - ** terms that appear in the document set. Each subsequent index in aIndex[] - ** is an index of prefixes of a specific length. - ** - ** Variable nPendingData contains an estimate the memory consumed by the - ** pending data structures, including hash table overhead, but not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash - ** tables are flushed to disk. Variable iPrevDocid is the docid of the most - ** recently inserted record. - */ - int nIndex; /* Size of aIndex[] */ - struct Fts3Index { - int nPrefix; /* Prefix length (0 for main terms index) */ - Fts3Hash hPending; /* Pending terms table for this index */ - } *aIndex; - int nMaxPendingData; /* Max pending data before flush to disk */ - int nPendingData; /* Current bytes of pending data */ - sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ - int iPrevLangid; /* Langid of recently inserted document */ - -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) - /* State variables used for validating that the transaction control - ** methods of the virtual table are called at appropriate times. These - ** values do not contribute to FTS functionality; they are used for - ** verifying the operation of the SQLite core. - */ - int inTransaction; /* True after xBegin but before xCommit/xRollback */ - int mxSavepoint; /* Largest valid xSavepoint integer */ -#endif - -#ifdef SQLITE_TEST - /* True to disable the incremental doclist optimization. This is controled - ** by special insert command 'test-no-incr-doclist'. */ - int bNoIncrDoclist; -#endif -}; +static int nodeHash(i64 iNode){ + return iNode % HASHSIZE; +} /* -** When the core wants to read from the virtual table, it creates a -** virtual table cursor (an instance of the following structure) using -** the xOpen method. Cursors are destroyed using the xClose method. +** Search the node hash table for node iNode. If found, return a pointer +** to it. Otherwise, return 0. */ -struct Fts3Cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - i16 eSearch; /* Search strategy (see below) */ - u8 isEof; /* True if at End Of Results */ - u8 isRequireSeek; /* True if must seek pStmt to %_content row */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - int iLangid; /* Language being queried for */ - int nPhrase; /* Number of matchable phrases in query */ - Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ - sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ - char *pNextId; /* Pointer into the body of aDoclist */ - char *aDoclist; /* List of docids for full-text queries */ - int nDoclist; /* Size of buffer at aDoclist */ - u8 bDesc; /* True to sort in descending order */ - int eEvalmode; /* An FTS3_EVAL_XX constant */ - int nRowAvg; /* Average size of database rows, in pages */ - sqlite3_int64 nDoc; /* Documents in table */ - i64 iMinDocid; /* Minimum docid to return */ - i64 iMaxDocid; /* Maximum docid to return */ - int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - u32 *aMatchinfo; /* Information about most recent match */ - int nMatchinfo; /* Number of elements in aMatchinfo[] */ - char *zMatchinfo; /* Matchinfo specification */ -}; - -#define FTS3_EVAL_FILTER 0 -#define FTS3_EVAL_NEXT 1 -#define FTS3_EVAL_MATCHINFO 2 +static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ + RtreeNode *p; + for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); + return p; +} /* -** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to -** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index -** of the column to be searched. For example, in -** -** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); -** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** -** Because the LHS of the MATCH operator is 2nd column "b", -** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" -** indicating that all columns should be searched, -** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +** Add node pNode to the node hash table. */ -#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ -#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ -#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ +static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; +} /* -** The lower 16-bits of the sqlite3_index_info.idxNum value set by -** the xBestIndex() method contains the Fts3Cursor.eSearch value described -** above. The upper 16-bits contain a combination of the following -** bits, used to describe extra constraints on full-text searches. +** Remove node pNode from the node hash table. */ -#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ -#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ -#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ - -struct Fts3Doclist { - char *aAll; /* Array containing doclist (or NULL) */ - int nAll; /* Size of a[] in bytes */ - char *pNextDocid; /* Pointer to next docid */ +static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode **pp; + if( pNode->iNode!=0 ){ + pp = &pRtree->aHash[nodeHash(pNode->iNode)]; + for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } + *pp = pNode->pNext; + pNode->pNext = 0; + } +} - sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ - int bFreeList; /* True if pList should be sqlite3_free()d */ - char *pList; /* Pointer to position list following iDocid */ - int nList; /* Length of position list */ -}; +/* +** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), +** indicating that node has not yet been assigned a node number. It is +** assigned a node number when nodeWrite() is called to write the +** node contents out to the database. +*/ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ + RtreeNode *pNode; + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); + if( pNode ){ + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->pParent = pParent; + pNode->isDirty = 1; + nodeReference(pParent); + } + return pNode; +} /* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in double-quotes (i.e. "one two three") -** nToken will be the number of tokens in the string. +** Obtain a reference to an r-tree node. */ -struct Fts3PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer z */ - int isPrefix; /* True if token ends with a "*" character */ - int bFirst; /* True if token must appear at position 0 */ +static int nodeAcquire( + Rtree *pRtree, /* R-tree structure */ + i64 iNode, /* Node number to load */ + RtreeNode *pParent, /* Either the parent node or NULL */ + RtreeNode **ppNode /* OUT: Acquired node */ +){ + int rc; + int rc2 = SQLITE_OK; + RtreeNode *pNode; - /* Variables above this point are populated when the expression is - ** parsed (by code in fts3_expr.c). Below this point the variables are - ** used when evaluating the expression. */ - Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ - Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ -}; + /* Check if the requested node is already in the hash table. If so, + ** increase its reference count and return it. + */ + if( (pNode = nodeHashLookup(pRtree, iNode)) ){ + assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); + if( pParent && !pNode->pParent ){ + nodeReference(pParent); + pNode->pParent = pParent; + } + pNode->nRef++; + *ppNode = pNode; + return SQLITE_OK; + } -struct Fts3Phrase { - /* Cache of doclist for this phrase. */ - Fts3Doclist doclist; - int bIncr; /* True if doclist is loaded incrementally */ - int iDoclistToken; + sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); + rc = sqlite3_step(pRtree->pReadNode); + if( rc==SQLITE_ROW ){ + const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); + } + } + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; - /* Variables below this point are populated by fts3_expr.c when parsing - ** a MATCH expression. Everything above is part of the evaluation phase. + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ -}; + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT_VTAB; + } + } -/* -** A tree of these objects forms the RHS of a MATCH operator. -** -** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist -** points to a malloced buffer, size nDoclist bytes, containing the results -** of this phrase query in FTS3 doclist format. As usual, the initial -** "Length" field found in doclists stored on disk is omitted from this -** buffer. -** -** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global -** matchinfo data. If it is not NULL, it points to an array of size nCol*3, -** where nCol is the number of columns in the queried FTS table. The array -** is populated as follows: -** -** aMI[iCol*3 + 0] = Undefined -** aMI[iCol*3 + 1] = Number of occurrences -** aMI[iCol*3 + 2] = Number of rows containing at least one instance -** -** The aMI array is allocated using sqlite3_malloc(). It should be freed -** when the expression node is. -*/ -struct Fts3Expr { - int eType; /* One of the FTSQUERY_XXX values defined below */ - int nNear; /* Valid if eType==FTSQUERY_NEAR */ - Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT_VTAB. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT_VTAB; + } + } - /* The following are used by the fts3_eval.c module. */ - sqlite3_int64 iDocid; /* Current docid */ - u8 bEof; /* True this expression is at EOF already */ - u8 bStart; /* True if iDocid is valid */ - u8 bDeferred; /* True if this expression is entirely deferred */ + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT_VTAB; + } + *ppNode = pNode; + }else{ + sqlite3_free(pNode); + *ppNode = 0; + } - u32 *aMI; -}; + return rc; +} /* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: -** -** "a OR b AND c NOT d NEAR e" -** -** is equivalent to: -** -** "a OR (b AND (c NOT (d NEAR e)))" +** Overwrite cell iCell of node pNode with the contents of pCell. */ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - - -/* fts3_write.c */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, - sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( - Fts3Table*,int,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); +static void nodeOverwriteCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ +){ + int ii; + u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + p += writeInt64(p, pCell->iRowid); + for(ii=0; ii<(pRtree->nDim*2); ii++){ + p += writeCoord(p, &pCell->aCoord[ii]); + } + pNode->isDirty = 1; +} -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); +/* +** Remove the cell with index iCell from node pNode. +*/ +static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ + u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + u8 *pSrc = &pDst[pRtree->nBytesPerCell]; + int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; + memmove(pDst, pSrc, nByte); + writeInt16(&pNode->zData[2], NCELL(pNode)-1); + pNode->isDirty = 1; +} -#ifndef SQLITE_DISABLE_FTS4_DEFERRED -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#else -# define sqlite3Fts3FreeDeferredTokens(x) -# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK -# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK -# define sqlite3Fts3FreeDeferredDoclists(x) -# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK -#endif +/* +** Insert the contents of cell pCell into node pNode. If the insert +** is successful, return SQLITE_OK. +** +** If there is not enough free space in pNode, return SQLITE_FULL. +*/ +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ +){ + int nCell; /* Current number of cells in pNode */ + int nMaxCell; /* Maximum number of cells for pNode */ -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; + nCell = NCELL(pNode); -/* Special values interpreted by sqlite3SegReaderCursor() */ -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 + assert( nCell<=nMaxCell ); + if( nCellzData[2], nCell+1); + pNode->isDirty = 1; + } -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + return (nCell==nMaxCell); +} -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, - int, int, int, const char *, int, int, int, Fts3MultiSegReader *); +/* +** If the node is dirty, write it out to the database. +*/ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode->isDirty ){ + sqlite3_stmt *p = pRtree->pWriteNode; + if( pNode->iNode ){ + sqlite3_bind_int64(p, 1, pNode->iNode); + }else{ + sqlite3_bind_null(p, 1); + } + sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); + sqlite3_step(p); + pNode->isDirty = 0; + rc = sqlite3_reset(p); + if( pNode->iNode==0 && rc==SQLITE_OK ){ + pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); + nodeHashInsert(pRtree, pNode); + } + } + return rc; +} -/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ -#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 -#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 -#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 -#define FTS3_SEGMENT_PREFIX 0x00000008 -#define FTS3_SEGMENT_SCAN 0x00000010 -#define FTS3_SEGMENT_FIRST 0x00000020 +/* +** Release a reference to a node. If the node is dirty and the reference +** count drops to zero, the node data is written to the database. +*/ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode ){ + assert( pNode->nRef>0 ); + pNode->nRef--; + if( pNode->nRef==0 ){ + if( pNode->iNode==1 ){ + pRtree->iDepth = -1; + } + if( pNode->pParent ){ + rc = nodeRelease(pRtree, pNode->pParent); + } + if( rc==SQLITE_OK ){ + rc = nodeWrite(pRtree, pNode); + } + nodeHashDelete(pRtree, pNode); + sqlite3_free(pNode); + } + } + return rc; +} -/* Type passed as 4th argument to SegmentReaderIterate() */ -struct Fts3SegFilter { - const char *zTerm; - int nTerm; - int iCol; - int flags; -}; +/* +** Return the 64-bit integer value associated with cell iCell of +** node pNode. If pNode is a leaf node, this is a rowid. If it is +** an internal node, then the 64-bit integer is a child page number. +*/ +static i64 nodeGetRowid( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ +){ + assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); +} -struct Fts3MultiSegReader { - /* Used internally by sqlite3Fts3SegReaderXXX() calls */ - Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ - int nSegment; /* Size of apSegment array */ - int nAdvance; /* How many seg-readers to advance */ - Fts3SegFilter *pFilter; /* Pointer to filter object */ - char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ +/* +** Return coordinate iCoord from cell iCell in node pNode. +*/ +static void nodeGetCoord( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ +){ + readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); +} - int iColFilter; /* If >=0, filter for this column */ - int bRestart; +/* +** Deserialize cell iCell of node pNode. Populate the structure pointed +** to by pCell with the results. +*/ +static void nodeGetCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + RtreeCoord *pCoord; + int ii; + pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pCoord = pCell->aCoord; + for(ii=0; iinDim*2; ii++){ + readCoord(&pData[ii*4], &pCoord[ii]); + } +} - /* Used by fts3.c only. */ - int nCost; /* Cost of running iterator */ - int bLookup; /* True if a lookup of a single entry. */ - /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ - char *zTerm; /* Pointer to term buffer */ - int nTerm; /* Size of zTerm in bytes */ - char *aDoclist; /* Pointer to doclist buffer */ - int nDoclist; /* Size of aDoclist[] in bytes */ -}; +/* Forward declaration for the function that does the work of +** the virtual table module xCreate() and xConnect() methods. +*/ +static int rtreeInit( + sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int +); -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); +/* +** Rtree virtual table module xCreate method. +*/ +static int rtreeCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} -#define fts3GetVarint32(p, piVal) ( \ - (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ -) +/* +** Rtree virtual table module xConnect method. +*/ +static int rtreeConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} -/* fts3.c */ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); -SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +/* +** Increment the r-tree reference count. +*/ +static void rtreeReference(Rtree *pRtree){ + pRtree->nBusy++; +} -/* fts3_tokenizer.c */ -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, - sqlite3_tokenizer **, char ** -); -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); +/* +** Decrement the r-tree reference count. When the reference count reaches +** zero the structure is deleted. +*/ +static void rtreeRelease(Rtree *pRtree){ + pRtree->nBusy--; + if( pRtree->nBusy==0 ){ + sqlite3_finalize(pRtree->pReadNode); + sqlite3_finalize(pRtree->pWriteNode); + sqlite3_finalize(pRtree->pDeleteNode); + sqlite3_finalize(pRtree->pReadRowid); + sqlite3_finalize(pRtree->pWriteRowid); + sqlite3_finalize(pRtree->pDeleteRowid); + sqlite3_finalize(pRtree->pReadParent); + sqlite3_finalize(pRtree->pWriteParent); + sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_free(pRtree); + } +} -/* fts3_snippet.c */ -SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, - const char *, const char *, int, int -); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +/* +** Rtree virtual table module xDisconnect method. +*/ +static int rtreeDisconnect(sqlite3_vtab *pVtab){ + rtreeRelease((Rtree *)pVtab); + return SQLITE_OK; +} -/* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr **, char ** -); -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); -#endif +/* +** Rtree virtual table module xDestroy method. +*/ +static int rtreeDestroy(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + int rc; + char *zCreate = sqlite3_mprintf( + "DROP TABLE '%q'.'%q_node';" + "DROP TABLE '%q'.'%q_rowid';" + "DROP TABLE '%q'.'%q_parent';", + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName + ); + if( !zCreate ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + } + if( rc==SQLITE_OK ){ + rtreeRelease(pRtree); + } -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, - sqlite3_tokenizer_cursor ** -); + return rc; +} -/* fts3_aux.c */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); +/* +** Rtree virtual table module xOpen method. +*/ +static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + RtreeCursor *pCsr; -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = pVTab; + rc = SQLITE_OK; + } + *ppCursor = (sqlite3_vtab_cursor *)pCsr; -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table*, Fts3MultiSegReader*, int, const char*, int); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + return rc; +} -/* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); -/* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); -#endif +/* +** Free the RtreeCursor.aConstraint[] array and its contents. +*/ +static void freeCursorConstraints(RtreeCursor *pCsr){ + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } +} -#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ -#endif /* _FTSINT_H */ +/* +** Rtree virtual table module xClose method. +*/ +static int rtreeClose(sqlite3_vtab_cursor *cur){ + Rtree *pRtree = (Rtree *)(cur->pVtab); + int ii; + RtreeCursor *pCsr = (RtreeCursor *)cur; + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr); + return SQLITE_OK; +} -/************** End of fts3Int.h *********************************************/ -/************** Continuing where we left off in fts3.c ***********************/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* +** Rtree virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int rtreeEof(sqlite3_vtab_cursor *cur){ + RtreeCursor *pCsr = (RtreeCursor *)cur; + return pCsr->atEOF; +} -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. +*/ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} #endif -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. +*/ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ +){ + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ -static int fts3EvalNext(Fts3Cursor *pCsr); -static int fts3EvalStart(Fts3Cursor *pCsr); -static int fts3TermSegReaderCursor( - Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -** The number of bytes written is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ - unsigned char *q = (unsigned char *) p; - sqlite_uint64 vu = v; - do{ - *q++ = (unsigned char) ((vu & 0x7f) | 0x80); - vu >>= 7; - }while( vu!=0 ); - q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); - return (int) (q - (unsigned char *)p); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; + } + } + return rc; } -#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ - v = (v & mask1) | ( (*ptr++) << shift ); \ - if( (v & mask2)==0 ){ var = v; return ret; } -#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ - v = (*ptr++); \ - if( (v & mask2)==0 ){ var = v; return ret; } - /* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. */ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; - u32 a; - u64 b; - int shift; +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); - b = (a & 0x0FFFFFFF ); + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); - for(shift=28; shift<=63; shift+=7){ - u64 c = *p++; - b += (c&0x7F) << shift; - if( (c & 0x80)==0 ) break; + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; } - *v = b; - return (int)(p - pStart); + *peWithin = NOT_WITHIN; } /* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a -** 32-bit integer before it is returned. +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. */ -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ - u32 a; +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ -#ifndef fts3GetVarint32 - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); -#else - a = (*p++); - assert( a & 0x80 ); -#endif + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); - a = (a & 0x0FFFFFFF ); - *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); - return 5; +/* +** One of the cells in node pNode is guaranteed to have a 64-bit +** integer value equal to iRowid. Return the index of this cell. +*/ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ + int ii; + int nCell = NCELL(pNode); + assert( nCell<200 ); + for(ii=0; ii>= 7; - }while( v!=0 ); - return i; +static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){ + RtreeNode *pParent = pNode->pParent; + if( pParent ){ + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + } + *piIndex = -1; + return SQLITE_OK; } /* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. -** -** Examples: -** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno +** Compare two search points. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second. ** +** The rScore is the primary key. Smaller rScore values come first. +** If the rScore is a tie, then use iLevel as the tie breaker with smaller +** iLevel values coming first. In this way, if rScore is the same for all +** SearchPoints, then iLevel becomes the deciding factor and the result +** is a depth-first search, which is the desired default behavior. */ -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ - char quote; /* Quote character (if any ) */ +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB +){ + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} - quote = z[0]; - if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ - int iIn = 1; /* Index of next byte to read from input */ - int iOut = 0; /* Index of next byte to write to output */ +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} - /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} - while( ALWAYS(z[iIn]) ){ - if( z[iIn]==quote ){ - if( z[iIn+1]!=quote ) break; - z[iOut++] = quote; - iIn += 2; - }else{ - z[iOut++] = z[iIn++]; - } - } - z[iOut] = '\0'; +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); } + return pCur->aNode[ii]; } /* -** Read a single varint from the doclist at *pp and advance *pp to point -** to the first byte past the end of the varint. Add the value of the varint -** to *pVal. +** Push a new element onto the priority queue */ -static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - *pVal += iVal; +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; } /* -** When this function is called, *pp points to the first byte following a -** varint that is part of a doclist (or position-list, or any other list -** of varints). This function moves *pp to point to the start of that varint, -** and sets *pVal by the varint value. -** -** Argument pStart points to the first byte of the doclist that the -** varint is part of. +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. */ -static void fts3GetReverseVarint( - char **pp, - char *pStart, - sqlite3_int64 *pVal +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ ){ - sqlite3_int64 iVal; - char *p; + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} - /* Pointer p now points at the first byte past the varint we are - ** interested in. So, unless the doclist is corrupt, the 0x80 bit is - ** clear on character p[-1]. */ - for(p = (*pp)-2; p>=pStart && *p&0x80; p--); - p++; - *pp = p; +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif - sqlite3Fts3GetVarint(p, &iVal); - *pVal = iVal; +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; } -/* -** The xDisconnect() virtual table method. +/* +** Rtree virtual table module xNext method. */ -static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int i; +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; - assert( p->nPendingData==0 ); - assert( p->pSegments==0 ); + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); + return rc; +} - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(p->aStmt[i]); +/* +** Rtree virtual table module xRowid method. +*/ +static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); } - sqlite3_free(p->zSegmentsTbl); - sqlite3_free(p->zReadExprlist); - sqlite3_free(p->zWriteExprlist); - sqlite3_free(p->zContentTbl); - sqlite3_free(p->zLanguageid); + return rc; +} - /* Invoke the tokenizer destructor to free the tokenizer. */ - p->pTokenizer->pModule->xDestroy(p->pTokenizer); +/* +** Rtree virtual table module xColumn method. +*/ +static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); - sqlite3_free(p); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 ){ + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); + }else{ + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + sqlite3_result_double(ctx, c.f); + }else +#endif + { + assert( pRtree->eCoordType==RTREE_COORD_INT32 ); + sqlite3_result_int(ctx, c.i); + } + } return SQLITE_OK; } -/* -** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is written -** into *pRc. -** -** If *pRc is initially non-zero then this routine is a no-op. +/* +** Use nodeAcquire() to obtain the leaf node containing the record with +** rowid iRowid. If successful, set *ppLeaf to point to the node and +** return SQLITE_OK. If there is no such record in the table, set +** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf +** to zero and return an SQLite error code. */ -static void fts3DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ ){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; + int rc; + *ppLeaf = 0; + sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); + if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ + i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; + rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); + sqlite3_reset(pRtree->pReadRowid); }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); + rc = sqlite3_reset(pRtree->pReadRowid); } + return rc; } /* -** The xDestroy() virtual table method. +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. */ -static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return code */ - const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ - sqlite3 *db = p->db; /* Database handle */ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ - /* Drop the shadow tables */ - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + /* Check that value is actually a blob. */ + if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlob<(int)sizeof(RtreeMatchArg) ){ + return SQLITE_ERROR; } - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); - /* If everything has worked, invoke fts3DisconnectMethod() to free the - ** memory associated with the Fts3Table structure and return SQLITE_OK. - ** Otherwise, return an SQLite error code. - */ - return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); -} + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + pBlob->nParam*sizeof(sqlite3_value*) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); + return SQLITE_ERROR; + } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; + pInfo->apSqlParam = pBlob->apSqlParam; -/* -** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table -** passed as the first argument. This is done as part of the xConnect() -** and xCreate() methods. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xFilter method. */ -static void fts3DeclareVtab(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ - const char *zLanguageid; +static int rtreeFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int ii; + int rc = SQLITE_OK; + int iCell = 0; - zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); - sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + rtreeReference(pRtree); - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", - zCols, p->zName, zLanguageid - ); - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); }else{ - rc = sqlite3_declare_vtab(p->db, zSql); + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ + pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); + pCsr->nConstraint = argc; + if( !pCsr->aConstraint ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); + for(ii=0; iiaConstraint[ii]; + p->op = idxStr[ii*2]; + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; + }else{ +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif + } + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } - - sqlite3_free(zSql); - sqlite3_free(zCols); - *pRc = rc; } + + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; } /* -** Create the %_stat table if it does not already exist. +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. */ -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ - fts3DbExec(pRc, p->db, - "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" - "(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); - if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif } /* -** Create the backing store tables (%_content, %_segments and %_segdir) -** required by the FTS3 table passed as the only argument. This is done -** as part of the vtab xCreate() method. +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): ** -** If the p->bHasDocsize boolean is true (indicating that this is an -** FTS4 table, not an FTS3 table) then also create the %_docsize and -** %_stat tables required by FTS4. +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 Unused Direct lookup by rowid. +** 2 See below R-tree query or full-table scan. +** ------------------------------------------------ +** +** If strategy 1 is used, then idxStr is not meaningful. If strategy +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to +** the constraint in sqlite3_index_info.aConstraintUsage[] with +** (argvIndex==1) etc. +** +** The first of each pair of bytes in idxStr identifies the constraint +** operator as follows: +** +** Operator Byte Value +** ---------------------- +** = 0x41 ('A') +** <= 0x42 ('B') +** < 0x43 ('C') +** >= 0x44 ('D') +** > 0x45 ('E') +** MATCH 0x46 ('F') +** ---------------------- +** +** The second of each pair of bytes identifies the coordinate column +** to which the constraint applies. The leftmost coordinate column +** is 'a', the second from the left 'b' etc. */ -static int fts3CreateTables(Fts3Table *p){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - sqlite3 *db = p->db; /* The database connection */ +static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; + int rc = SQLITE_OK; + int ii; + int bMatch = 0; /* True if there exists a MATCH constraint */ + i64 nRow; /* Estimated rows returned by this scan */ - if( p->zContentTbl==0 ){ - const char *zLanguageid = p->zLanguageid; - char *zContentCols; /* Columns of %_content table */ + int iIdx = 0; + char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; + memset(zIdxStr, 0, sizeof(zIdxStr)); - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zLanguageid && zContentCols ){ - zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + /* Check if there exists a MATCH constraint - even an unusable one. If there + ** is, do not consider the lookup-by-rowid plan as using such a plan would + ** require the VDBE to evaluate the MATCH constraint, which is not currently + ** possible. */ + for(ii=0; iinConstraint; ii++){ + if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + bMatch = 1; } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); } - /* Create other tables */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", - p->zDb, p->zName - ); - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segdir'(" - "level INTEGER," - "idx INTEGER," - "start_block INTEGER," - "leaves_end_block INTEGER," - "end_block INTEGER," - "root BLOB," - "PRIMARY KEY(level, idx)" - ");", - p->zDb, p->zName - ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", - p->zDb, p->zName - ); - } - assert( p->bHasStat==p->bFts4 ); - if( p->bHasStat ){ - sqlite3Fts3CreateStatTable(&rc, p); - } - return rc; -} + assert( pIdxInfo->idxStr==0 ); + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; -/* -** Store the current database page-size in bytes in p->nPgsz. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. -*/ -static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - char *zSql; /* SQL text "PRAGMA %Q.page_size" */ - sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - p->nPgsz = sqlite3_column_int(pStmt, 0); - rc = sqlite3_finalize(pStmt); - }else if( rc==SQLITE_AUTH ){ - p->nPgsz = 1024; - rc = SQLITE_OK; + if( bMatch==0 && p->usable + && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + /* We have an equality constraint on the rowid. Use strategy 1. */ + int jj; + for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; + pIdxInfo->aConstraintUsage[jj].omit = 0; } - } - assert( p->nPgsz>0 || rc!=SQLITE_OK ); - sqlite3_free(zSql); - *pRc = rc; - } -} + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[ii].argvIndex = 1; + pIdxInfo->aConstraintUsage[jj].omit = 1; -/* -** "Special" FTS4 arguments are column specifications of the following form: -** -** = -** -** There may not be whitespace surrounding the "=" character. The -** term may be quoted, but the may not. -*/ -static int fts3IsSpecialColumn( - const char *z, - int *pnKey, - char **pzValue -){ - char *zValue; - const char *zCsr = z; + /* This strategy involves a two rowid lookups on an B-Tree structures + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. + */ + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); + return SQLITE_OK; + } - while( *zCsr!='=' ){ - if( *zCsr=='\0' ) return 0; - zCsr++; + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + u8 op; + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + default: + assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; + } + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; + } } - *pnKey = (int)(zCsr-z); - zValue = sqlite3_mprintf("%s", &zCsr[1]); - if( zValue ){ - sqlite3Fts3Dequote(zValue); + pIdxInfo->idxNum = 2; + pIdxInfo->needToFreeIdxStr = 1; + if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ + return SQLITE_NOMEM; } - *pzValue = zValue; - return 1; + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + + return rc; } /* -** Append the output of a printf() style formatting to an existing string. +** Return the N-dimensional volumn of the cell stored in *p. */ -static void fts3Appendf( - int *pRc, /* IN/OUT: Error code */ - char **pz, /* IN/OUT: Pointer to string buffer */ - const char *zFormat, /* Printf format string to append */ - ... /* Arguments for printf format string */ -){ - if( *pRc==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( z && *pz ){ - char *z2 = sqlite3_mprintf("%s%s", *pz, z); - sqlite3_free(z); - z = z2; - } - if( z==0 ) *pRc = SQLITE_NOMEM; - sqlite3_free(*pz); - *pz = z; +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; + int ii; + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } + return area; } /* -** Return a copy of input string zInput enclosed in double-quotes (") and -** with all double quote characters escaped. For example: -** -** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" -** -** The pointer returned points to memory obtained from sqlite3_malloc(). It -** is the callers responsibility to call sqlite3_free() to release this -** memory. +** Return the margin length of cell p. The margin length is the sum +** of the objects size in each dimension. */ -static char *fts3QuoteId(char const *zInput){ - int nRet; - char *zRet; - nRet = 2 + (int)strlen(zInput)*2 + 1; - zRet = sqlite3_malloc(nRet); - if( zRet ){ - int i; - char *z = zRet; - *(z++) = '"'; - for(i=0; zInput[i]; i++){ - if( zInput[i]=='"' ) *(z++) = '"'; - *(z++) = zInput[i]; - } - *(z++) = '"'; - *(z++) = '\0'; +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = (RtreeDValue)0; + int ii; + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } - return zRet; + return margin; } /* -** Return a list of comma separated SQL expressions and a FROM clause that -** could be used in a SELECT statement such as the following: -** -** SELECT FROM %_content AS x ... -** -** to return the docid, followed by each column of text data in order -** from left to write. If parameter zFunc is not NULL, then instead of -** being returned directly each column of text data is passed to an SQL -** function named zFunc first. For example, if zFunc is "unzip" and the -** table has the three user-defined columns "a", "b", and "c", the following -** string is returned: -** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. +** Store the union of cells p1 and p2 in p1. */ -static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - - if( p->zContentTbl==0 ){ - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); +static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); + p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); } - sqlite3_free(zFree); }else{ - fts3Appendf(pRc, &zRet, "rowid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); + p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); } } - fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", - p->zDb, - (p->zContentTbl ? p->zContentTbl : p->zName), - (p->zContentTbl ? "" : "_content") - ); - return zRet; } /* -** Return a list of N comma separated question marks, where N is the number -** of columns in the %_content table (one for the docid plus one for each -** user-defined text column). -** -** If argument zFunc is not NULL, then all but the first question mark -** is preceded by zFunc and an open bracket, and followed by a closed -** bracket. For example, if zFunc is "zip" and the FTS3 table has three -** user-defined text columns, the following string is returned: -** -** "?, zip(?), zip(?), zip(?)" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. +** Return true if the area covered by p2 is a subset of the area covered +** by p1. False otherwise. */ -static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "?"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", ?"); +static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( (!isInt && (a2[0].fa1[1].f)) + || ( isInt && (a2[0].ia1[1].i)) + ){ + return 0; + } } - sqlite3_free(zFree); - return zRet; + return 1; } /* -** This function interprets the string at (*pp) as a non-negative integer -** value. It reads the integer and sets *pnOut to the value read, then -** sets *pp to point to the byte immediately following the last byte of -** the integer value. -** -** Only decimal digits ('0'..'9') may be part of an integer value. -** -** If *pp does not being with a decimal digit SQLITE_ERROR is returned and -** the output value undefined. Otherwise SQLITE_OK is returned. -** -** This function is used when parsing the "prefix=" FTS4 parameter. +** Return the amount cell p would grow by if it were unioned with pCell. */ -static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p; /* Iterator pointer */ - int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - } - if( p==*pp ) return SQLITE_ERROR; - *pnOut = nInt; - *pp = p; - return SQLITE_OK; +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; + RtreeCell cell; + memcpy(&cell, p, sizeof(RtreeCell)); + area = cellArea(pRtree, &cell); + cellUnion(pRtree, &cell, pCell); + return (cellArea(pRtree, &cell)-area); } -/* -** This function is called to allocate an array of Fts3Index structures -** representing the indexes maintained by the current FTS table. FTS tables -** always maintain the main "terms" index, but may also maintain one or -** more "prefix" indexes, depending on the value of the "prefix=" parameter -** (if any) specified as part of the CREATE VIRTUAL TABLE statement. -** -** Argument zParam is passed the value of the "prefix=" option if one was -** specified, or NULL otherwise. -** -** If no error occurs, SQLITE_OK is returned and *apIndex set to point to -** the allocated array. *pnIndex is set to the number of elements in the -** array. If an error does occur, an SQLite error code is returned. -** -** Regardless of whether or not an error is returned, it is the responsibility -** of the caller to call sqlite3_free() on the output array to free it. -*/ -static int fts3PrefixParameter( - const char *zParam, /* ABC in prefix=ABC parameter to parse */ - int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +static RtreeDValue cellOverlap( + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, + int nCell ){ - struct Fts3Index *aIndex; /* Allocated array */ - int nIndex = 1; /* Number of entries in array */ - - if( zParam && zParam[0] ){ - const char *p; - nIndex++; - for(p=zParam; *p; p++){ - if( *p==',' ) nIndex++; - } - } - - aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = aIndex; - *pnIndex = nIndex; - if( !aIndex ){ - return SQLITE_NOMEM; - } - - memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); - if( zParam ){ - const char *p = zParam; - int i; - for(i=1; inDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-iHeight); ii++){ + int iCell; + sqlite3_int64 iBest = 0; - if( rc==SQLITE_OK ){ - const char **azCol; /* Output array */ - int nStr = 0; /* Size of all column names (incl. 0x00) */ - int nCol; /* Number of table columns */ - int i; /* Used to iterate through columns */ + RtreeDValue fMinGrowth = RTREE_ZERO; + RtreeDValue fMinArea = RTREE_ZERO; - /* Loop through the returned columns. Set nStr to the number of bytes of - ** space required to store a copy of each column name, including the - ** nul-terminator byte. */ - nCol = sqlite3_column_count(pStmt); - for(i=0; i module name ("fts3" or "fts4") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. +** A cell with the same content as pCell has just been inserted into +** the node pNode. This function updates the bounding box cells in +** all ancestor elements. */ -static int fts3InitVtab( - int isCreate, /* True for xCreate, false for xConnect */ - sqlite3 *db, /* The SQLite database connection */ - void *pAux, /* Hash table containing tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ +static int AdjustTree( + Rtree *pRtree, /* Rtree table */ + RtreeNode *pNode, /* Adjust ancestry of this node. */ + RtreeCell *pCell /* This cell was just inserted */ ){ - Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p = 0; /* Pointer to allocated vtab */ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ - int iCol; /* Column index */ - int nString = 0; /* Bytes required to hold all column names */ - int nCol = 0; /* Number of columns in the FTS table */ - char *zCsr; /* Space for holding column names */ - int nDb; /* Bytes required to hold database name */ - int nName; /* Bytes required to hold table name */ - int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - const char **aCol; /* Array of column names */ - sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - - int nIndex; /* Size of aIndex[] array */ - struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ - - /* The results of parsing supported FTS4 key=value options: */ - int bNoDocsize = 0; /* True to omit %_docsize table */ - int bDescIdx = 0; /* True to store descending indexes */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ - char *zCompress = 0; /* compress=? parameter (or NULL) */ - char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zContent = 0; /* content=? parameter (or NULL) */ - char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ - char **azNotindexed = 0; /* The set of notindexed= columns */ - int nNotindexed = 0; /* Size of azNotindexed[] array */ - - assert( strlen(argv[0])==4 ); - assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) - || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) - ); - - nDb = (int)strlen(argv[1]) + 1; - nName = (int)strlen(argv[2]) + 1; - - nByte = sizeof(const char *) * (argc-2); - aCol = (const char **)sqlite3_malloc(nByte); - if( aCol ){ - memset((void*)aCol, 0, nByte); - azNotindexed = (char **)sqlite3_malloc(nByte); - } - if( azNotindexed ){ - memset(azNotindexed, 0, nByte); - } - if( !aCol || !azNotindexed ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - - /* Loop through all of the arguments passed by the user to the FTS3/4 - ** module (i.e. all the column names and special arguments). This loop - ** does the following: - ** - ** + Figures out the number of columns the FTSX table will have, and - ** the number of bytes of space that must be allocated to store copies - ** of the column names. - ** - ** + If there is a tokenizer specification included in the arguments, - ** initializes the tokenizer pTokenizer. - */ - for(i=3; rc==SQLITE_OK && ipParent ){ + RtreeNode *pParent = p->pParent; + RtreeCell cell; + int iCell; - /* Check if this is a tokenizer specification */ - if( !pTokenizer - && strlen(z)>8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) - && 0==sqlite3Fts3IsIdChar(z[8]) - ){ - rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT_VTAB; } - /* Check if it is an FTS4 special argument. */ - else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ - struct Fts4Option { - const char *zOpt; - int nOpt; - } aFts4Opt[] = { - { "matchinfo", 9 }, /* 0 -> MATCHINFO */ - { "prefix", 6 }, /* 1 -> PREFIX */ - { "compress", 8 }, /* 2 -> COMPRESS */ - { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ - { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 }, /* 6 -> LANGUAGEID */ - { "notindexed", 10 } /* 7 -> NOTINDEXED */ - }; + nodeGetCell(pRtree, pParent, iCell, &cell); + if( !cellContains(pRtree, &cell, pCell) ){ + cellUnion(pRtree, &cell, pCell); + nodeOverwriteCell(pRtree, pParent, &cell, iCell); + } + + p = pParent; + } + return SQLITE_OK; +} - int iOpt; - if( !zVal ){ - rc = SQLITE_NOMEM; - }else{ - for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ - break; - } - } - if( iOpt==SizeofArray(aFts4Opt) ){ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - }else{ - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; +/* +** Write mapping (iRowid->iNode) to the _rowid table. +*/ +static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ + sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); + sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); + sqlite3_step(pRtree->pWriteRowid); + return sqlite3_reset(pRtree->pWriteRowid); +} - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; +/* +** Write mapping (iNode->iPar) to the _parent table. +*/ +static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ + sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); + sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); + sqlite3_step(pRtree->pWriteParent); + return sqlite3_reset(pRtree->pWriteParent); +} - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; +static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; +/* +** Arguments aIdx, aDistance and aSpare all point to arrays of size +** nIdx. The aIdx array contains the set of integers from 0 to +** (nIdx-1) in no particular order. This function sorts the values +** in aIdx according to the indexed values in aDistance. For +** example, assuming the inputs: +** +** aIdx = { 0, 1, 2, 3 } +** aDistance = { 5.0, 2.0, 7.0, 6.0 } +** +** this function sets the aIdx array to contain: +** +** aIdx = { 0, 1, 2, 3 } +** +** The aSpare array is used as temporary working space by the +** sorting algorithm. +*/ +static void SortByDistance( + int *aIdx, + int nIdx, + RtreeDValue *aDistance, + int *aSpare +){ + if( nIdx>1 ){ + int iLeft = 0; + int iRight = 0; - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; + SortByDistance(aLeft, nLeft, aDistance, aSpare); + SortByDistance(aRight, nRight, aDistance, aSpare); - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; - } + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + + while( iLeft1 ){ - /* If a languageid= option was specified, remove the language id - ** column from the aCol[] array. */ - if( rc==SQLITE_OK && zLanguageid ){ - int j; - for(j=0; jnDim+1)*(sizeof(int*)+nCell*sizeof(int)); + + aaSorted = (int **)sqlite3_malloc(nByte); + if( !aaSorted ){ + return SQLITE_NOMEM; } - memset(p, 0, nByte); - p->db = db; - p->nColumn = nCol; - p->nPendingData = 0; - p->azColumn = (char **)&p[1]; - p->pTokenizer = pTokenizer; - p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; - p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ - p->zContentTbl = zContent; - p->zLanguageid = zLanguageid; - zContent = 0; - zLanguageid = 0; - TESTONLY( p->inTransaction = -1 ); - TESTONLY( p->mxSavepoint = -1 ); - p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; - memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); - p->nIndex = nIndex; - for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; + memset(aaSorted, 0, nByte); + for(ii=0; iinDim; ii++){ + int jj; + aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; + for(jj=0; jjabNotindexed = (u8 *)&p->aIndex[nIndex]; - /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->abNotindexed[nCol]; - p->zName = zCsr; - memcpy(zCsr, argv[2], nName); - zCsr += nName; - p->zDb = zCsr; - memcpy(zCsr, argv[1], nDb); - zCsr += nDb; + for(ii=0; iinDim; ii++){ + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; + int iBestLeft = 0; + int nLeft; - /* Fill in the azColumn array */ - for(iCol=0; iColazColumn[iCol] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } + for( + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft++ + ){ + RtreeCell left; + RtreeCell right; + int kk; + RtreeDValue overlap; + RtreeDValue area; - /* Fill in the abNotindexed array */ - for(iCol=0; iColazColumn[iCol]); - for(i=0; iazColumn[iCol], zNot, n) + memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); + memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); + for(kk=1; kk<(nCell-1); kk++){ + if( kkabNotindexed[iCol] = 1; - sqlite3_free(zNot); - azNotindexed[i] = 0; + iBestLeft = nLeft; + fBestOverlap = overlap; + fBestArea = area; } } - } - for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); - p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); - if( rc!=SQLITE_OK ) goto fts3_init_out; - /* If this is an xCreate call, create the underlying tables in the - ** database. TODO: For xConnect(), it could verify that said tables exist. - */ - if( isCreate ){ - rc = fts3CreateTables(p); + if( ii==0 || marginbHasStat = 2; + memcpy(pBboxLeft, &aCell[aaSorted[iBestDim][0]], sizeof(RtreeCell)); + memcpy(pBboxRight, &aCell[aaSorted[iBestDim][iBestSplit]], sizeof(RtreeCell)); + for(ii=0; iinNodeSize = p->nPgsz-35; + sqlite3_free(aaSorted); + return SQLITE_OK; +} - /* Declare the table schema to SQLite. */ - fts3DeclareVtab(&rc, p); -fts3_init_out: - sqlite3_free(zPrefix); - sqlite3_free(aIndex); - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - sqlite3_free(zContent); - sqlite3_free(zLanguageid); - for(i=0; ipModule->xDestroy(pTokenizer); +static int updateMapping( + Rtree *pRtree, + i64 iRowid, + RtreeNode *pNode, + int iHeight +){ + int (*xSetMapping)(Rtree *, sqlite3_int64, sqlite3_int64); + xSetMapping = ((iHeight==0)?rowidWrite:parentWrite); + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; } - }else{ - assert( p->pSegments==0 ); - *ppVTab = &p->base; } - return rc; + return xSetMapping(pRtree, iRowid, pNode->iNode); } -/* -** The xConnect() and xCreate() methods for the virtual table. All the -** work is done in function fts3InitVtab(). -*/ -static int fts3ConnectMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); -} -static int fts3CreateMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ +static int SplitNode( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight ){ - return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); -} + int i; + int newCellIsRight = 0; -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + RtreeCell *aCell; + int *aiUsed; -/* -** Implementation of the xBestIndex method for FTS3 tables. There -** are three possible strategies, in order of preference: -** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. -*/ -static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - Fts3Table *p = (Fts3Table *)pVTab; - int i; /* Iterator variable */ - int iCons = -1; /* Index of constraint to use */ + RtreeNode *pLeft = 0; + RtreeNode *pRight = 0; - int iLangidCons = -1; /* Index of langid=x constraint, if present */ - int iDocidGe = -1; /* Index of docid>=x constraint, if present */ - int iDocidLe = -1; /* Index of docid<=x constraint, if present */ - int iIdx; + RtreeCell leftbbox; + RtreeCell rightbbox; - /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient - ** strategy is possible. + /* Allocate an array and populate it with a copy of pCell and + ** all cells from node pLeft. Then zero the original node. */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 5000000; - for(i=0; inConstraint; i++){ - int bDocid; /* True if this constraint is on docid */ - struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ){ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* There exists an unusable MATCH constraint. This means that if - ** the planner does elect to use the results of this call as part - ** of the overall query plan the user will see an "unable to use - ** function MATCH in the requested context" error. To discourage - ** this, return a very high cost here. */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 1e50; - fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); - return SQLITE_OK; - } - continue; - } + aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + if( !aCell ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + aiUsed = (int *)&aCell[nCell+1]; + memset(aiUsed, 0, sizeof(int)*(nCell+1)); + for(i=0; iiColumn<0 || pCons->iColumn==p->nColumn+1); + if( pNode->iNode==1 ){ + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); + pRtree->iDepth++; + pNode->isDirty = 1; + writeInt16(pNode->zData, pRtree->iDepth); + }else{ + pLeft = pNode; + pRight = nodeNew(pRtree, pLeft->pParent); + nodeReference(pLeft); + } - /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ - pInfo->idxNum = FTS3_DOCID_SEARCH; - pInfo->estimatedCost = 1.0; - iCons = i; - } + if( !pLeft || !pRight ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } - /* A MATCH constraint. Use a full-text search. - ** - ** If there is more than one MATCH constraint available, use the first - ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even - ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested - ** context" error. - */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH - && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn - ){ - pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; - pInfo->estimatedCost = 2.0; - iCons = i; - } + memset(pLeft->zData, 0, pRtree->iNodeSize); + memset(pRight->zData, 0, pRtree->iNodeSize); - /* Equality constraint on the langid column */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && pCons->iColumn==p->nColumn + 2 - ){ - iLangidCons = i; - } + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } - if( bDocid ){ - switch( pCons->op ){ - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: - iDocidGe = i; - break; + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) + || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) + ){ + goto splitnode_out; + } - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: - iDocidLe = i; - break; - } + rightbbox.iRowid = pRight->iNode; + leftbbox.iRowid = pLeft->iNode; + + if( pNode->iNode==1 ){ + rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + }else{ + RtreeNode *pParent = pLeft->pParent; + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; } } + if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ + goto splitnode_out; + } - iIdx = 1; - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; - pInfo->aConstraintUsage[iCons].omit = 1; - } - if( iLangidCons>=0 ){ - pInfo->idxNum |= FTS3_HAVE_LANGID; - pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; - } - if( iDocidGe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_GE; - pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; - } - if( iDocidLe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_LE; - pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; - } - - /* Regardless of the strategy selected, FTS can deliver rows in rowid (or - ** docid) order. Both ascending and descending are possible. - */ - if( pInfo->nOrderBy==1 ){ - struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ - if( pOrder->desc ){ - pInfo->idxStr = "DESC"; - }else{ - pInfo->idxStr = "ASC"; + for(i=0; iiRowid ){ + newCellIsRight = 1; + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( pNode->iNode==1 ){ + for(i=0; iorderByConsumed = 1; } + }else if( newCellIsRight==0 ){ + rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); } - assert( p->pSegments==0 ); - return SQLITE_OK; -} - -/* -** Implementation of xOpen method. -*/ -static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ - - UNUSED_PARAMETER(pVTab); - - /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, - ** if the allocation fails, return SQLITE_NOMEM. - */ - *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); - if( !pCsr ){ - return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRight); + pRight = 0; + } + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pLeft); + pLeft = 0; } - memset(pCsr, 0, sizeof(Fts3Cursor)); - return SQLITE_OK; -} -/* -** Close the cursor. For additional information see the documentation -** on the xClose method of the virtual table interface. -*/ -static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); - sqlite3Fts3ExprFree(pCsr->pExpr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_free(pCsr); - return SQLITE_OK; +splitnode_out: + nodeRelease(pRtree, pRight); + nodeRelease(pRtree, pLeft); + sqlite3_free(aCell); + return rc; } /* -** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then -** compose and prepare an SQL statement of the form: -** -** "SELECT FROM %_content WHERE rowid = ?" -** -** (or the equivalent for a content=xxx table) and set pCsr->pStmt to -** it. If an error occurs, return an SQLite error code. +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. ** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ +static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ int rc = SQLITE_OK; - if( pCsr->pStmt==0 ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; + pChild = pChild->pParent; } - *ppStmt = pCsr->pStmt; return rc; } -/* -** Position the pCsr->pStmt statement so that it is on the row -** of the %_content table that contains the last match. Return -** SQLITE_OK on success. -*/ -static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ - int rc = SQLITE_OK; - if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; +static int deleteCell(Rtree *, RtreeNode *, int, int); - rc = fts3CursorSeekStmt(pCsr, &pStmt); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occurred, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. */ - rc = FTS_CORRUPT_VTAB; - pCsr->isEof = 1; - } - } - } +static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ + int rc; + int rc2; + RtreeNode *pParent = 0; + int iCell; + + assert( pNode->nRef==1 ); + + /* Remove the entry in the parent cell. */ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ + return rc; } - if( rc!=SQLITE_OK && pContext ){ - sqlite3_result_error_code(pContext, rc); + /* Remove the xxx_node entry. */ + sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteNode); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ + return rc; + } + + /* Remove the xxx_parent entry. */ + sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteParent); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ + return rc; + } + + /* Remove the node from the in-memory hash table and link it into + ** the Rtree.pDeleted list. Its contents will be re-inserted later on. + */ + nodeHashDelete(pRtree, pNode); + pNode->iNode = iHeight; + pNode->pNext = pRtree->pDeleted; + pNode->nRef++; + pRtree->pDeleted = pNode; + + return SQLITE_OK; +} + +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; + if( pParent ){ + int ii; + int nCell = NCELL(pNode); + RtreeCell box; /* Bounding box for pNode */ + nodeGetCell(pRtree, pNode, 0, &box); + for(ii=1; iiiNode; + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } } return rc; } /* -** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this -** function via the zNode/nNode parameters. The term to search for is -** passed in zTerm/nTerm. -** -** If piFirst is not NULL, then this function sets *piFirst to the blockid -** of the child node that heads the sub-tree that may contain the term. -** -** If piLast is not NULL, then *piLast is set to the right-most child node -** that heads a sub-tree that may contain a term for which zTerm/nTerm is -** a prefix. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +** Delete the cell at index iCell of node pNode. After removing the +** cell, adjust the r-tree data structure if required. */ -static int fts3ScanInteriorNode( - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piFirst, /* OUT: Selected child node */ - sqlite3_int64 *piLast /* OUT: Selected child node */ -){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ +static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; + int rc; - /* Skip over the 'height' varint that occurs at the start of every - ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. - ** - ** Even if the data structure on disk is corrupted, this (reading two - ** varints from the buffer) does not risk an overread. If zNode is a - ** root node, then the buffer comes from a SELECT statement. SQLite does - ** not make this guarantee explicitly, but in practice there are always - ** either more than 20 bytes of allocated space following the nNode bytes of - ** contents, or two zero bytes. Or, if the node is read from the %_segments - ** table, then there are always 20 bytes of zeroed padding following the - ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). - */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - if( zCsr>zEnd ){ - return FTS_CORRUPT_VTAB; + if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ + return rc; } - - while( zCsrzEnd ){ - rc = FTS_CORRUPT_VTAB; - goto finish_scan; - } - if( nPrefix+nSuffix>nAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - rc = SQLITE_NOMEM; - goto finish_scan; - } - zBuffer = zNew; - } - assert( zBuffer ); - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; - piFirst = 0; - } + /* Remove the cell from the node. This call just moves bytes around + ** the in-memory node image, so it cannot fail. + */ + nodeDeleteCell(pRtree, pNode, iCell); - if( piLast && cmp<0 ){ - *piLast = iChild; - piLast = 0; + /* If the node is not the tree root and now has less than the minimum + ** number of cells, remove it from the tree. Otherwise, update the + ** cell in the parent node so that it tightly contains the updated + ** node. + */ + pParent = pNode->pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)1 ){ - char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob; /* Size of zBlob in bytes */ + /* Allocate the buffers used by this operation. The allocation is + ** relinquished before this function returns. + */ + aCell = (RtreeCell *)sqlite3_malloc(n * ( + sizeof(RtreeCell) + /* aCell array */ + sizeof(int) + /* aOrder array */ + sizeof(int) + /* aSpare array */ + sizeof(RtreeDValue) /* aDistance array */ + )); + if( !aCell ){ + return SQLITE_NOMEM; + } + aOrder = (int *)&aCell[n]; + aSpare = (int *)&aOrder[n]; + aDistance = (RtreeDValue *)&aSpare[n]; - if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + for(ii=0; iinDim; iDim++){ + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + } + } + for(iDim=0; iDimnDim; iDim++){ + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); + } + + for(ii=0; iinDim; iDim++){ + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); + aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); + } + } + + SortByDistance(aOrder, nCell, aDistance, aSpare); + nodeZero(pRtree, pNode); + + for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ + RtreeCell *p = &aCell[aOrder[ii]]; + nodeInsertCell(pRtree, pNode, p); + if( p->iRowid==pCell->iRowid ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, p->iRowid, pNode->iNode); + } + } + } + if( rc==SQLITE_OK ){ + rc = fixBoundingBox(pRtree, pNode); + } + for(; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + RtreeNode *pInsert; + RtreeCell *p = &aCell[aOrder[ii]]; + rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); + rc2 = nodeRelease(pRtree, pInsert); if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + rc = rc2; } - sqlite3_free(zBlob); - piLeaf = 0; - zBlob = 0; } + } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + sqlite3_free(aCell); + return rc; +} + +/* +** Insert cell pCell into node pNode. Node pNode is the head of a +** subtree iHeight high (leaf nodes have iHeight==0). +*/ +static int rtreeInsertCell( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int rc = SQLITE_OK; + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + if( nodeInsertCell(pRtree, pNode, pCell) ){ + if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ + rc = SplitNode(pRtree, pNode, pCell, iHeight); + }else{ + pRtree->iReinsertHeight = iHeight; + rc = Reinsert(pRtree, pNode, pCell, iHeight); } + }else{ + rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } } - sqlite3_free(zBlob); } + return rc; +} + +static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ + int ii; + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + + for(ii=0; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } return rc; } /* -** This function is used to create delta-encoded serialized lists of FTS3 -** varints. Each call to this function appends a single varint to a list. +** Select a currently unused rowid for a new r-tree record. */ -static void fts3PutDeltaVarint( - char **pp, /* IN/OUT: Output pointer */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - sqlite3_int64 iVal /* Write this value to the list */ -){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); - *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); - *piPrev = iVal; +static int newRowid(Rtree *pRtree, i64 *piRowid){ + int rc; + sqlite3_bind_null(pRtree->pWriteRowid, 1); + sqlite3_bind_null(pRtree->pWriteRowid, 2); + sqlite3_step(pRtree->pWriteRowid); + rc = sqlite3_reset(pRtree->pWriteRowid); + *piRowid = sqlite3_last_insert_rowid(pRtree->db); + return rc; } /* -** When this function is called, *ppPoslist is assumed to point to the -** start of a position-list. After it returns, *ppPoslist points to the -** first byte after the position-list. -** -** A position list is list of positions (delta encoded) and columns for -** a single document record of a doclist. So, in other words, this -** routine advances *ppPoslist so that it points to the next docid in -** the doclist, or to the first byte past the end of the doclist. -** -** If pp is not NULL, then the contents of the position list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. +** Remove the entry with rowid=iDelete from the r-tree structure. */ -static void fts3PoslistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ - /* The end of a position list is marked by a zero encoded as an FTS3 - ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by - ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail - ** of some other, multi-byte, value. - ** - ** The following while-loop moves pEnd to point to the first byte that is not - ** immediately preceded by a byte with the 0x80 bit set. Then increments - ** pEnd once more so that it points to the byte immediately following the - ** last byte in the position-list. + + /* Obtain a reference to the root node to initialize Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. */ - while( *pEnd | c ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && (*pEnd)==0 ); + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } - pEnd++; /* Advance past the POS_END terminator byte */ - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } } - *ppPoslist = pEnd; -} -/* -** When this function is called, *ppPoslist is assumed to point to the -** start of a column-list. After it returns, *ppPoslist points to the -** to the terminator (POS_COLUMN or POS_END) byte of the column-list. -** -** A column-list is list of delta-encoded positions for a single column -** within a single document within a doclist. -** -** The column-list is terminated either by a POS_COLUMN varint (1) or -** a POS_END varint (0). This routine leaves *ppPoslist pointing to -** the POS_COLUMN or POS_END that terminates the column-list. -** -** If pp is not NULL, then the contents of the column-list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. The POS_COLUMN or POS_END terminator -** is not copied into *pp. -*/ -static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } - /* A column-list is terminated by either a 0x01 or 0x00 byte that is - ** not part of a multi-byte varint. + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && ((*pEnd)&0xfe)==0 ); - } - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } } - *ppPoslist = pEnd; -} -/* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. -*/ -#define POSITION_LIST_END 0x7fffffff + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } -/* -** This function is used to help parse position-lists. When this function is -** called, *pp may point to the start of the next varint in the position-list -** being parsed, or it may point to 1 byte past the end of the position-list -** (in which case **pp will be a terminator bytes POS_END (0) or -** (1)). -** -** If *pp points past the end of the current position-list, set *pi to -** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, -** increment the current value of *pi by the value read, and set *pp to -** point to the next value before returning. -** -** Before calling this routine *pi must be initialized to the value of -** the previous position, or zero if we are reading the first position -** in the position-list. Because positions are delta-encoded, the value -** of the previous position is needed in order to compute the value of -** the next position. -*/ -static void fts3ReadNextPos( - char **pp, /* IN/OUT: Pointer into position-list buffer */ - sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ -){ - if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); - *pi -= 2; + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); }else{ - *pi = POSITION_LIST_END; + nodeRelease(pRtree, pRoot); } + + return rc; } /* -** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by -** the value of iCol encoded as a varint to *pp. This will start a new -** column list. -** -** Set *pp to point to the byte just after the last byte written before -** returning (do not modify it if iCol==0). Return the total number of bytes -** written (0 if iCol==0). +** Rounding constants for float->double conversion. */ -static int fts3PutColNumber(char **pp, int iCol){ - int n = 0; /* Number of bytes written */ - if( iCol ){ - char *p = *pp; /* Output pointer */ - n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); - *p = 0x01; - *pp = &p[n]; - } - return n; -} +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ +#if !defined(SQLITE_RTREE_INT_ONLY) /* -** Compute the union of two position lists. The output written -** into *pp contains all positions of both *pp1 and *pp2 in sorted -** order and with any duplicates removed. All pointers are -** updated appropriately. The caller is responsible for insuring -** that there is enough space in *pp to hold the complete output. +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. */ -static void fts3PoslistMerge( - char **pp, /* Output buffer */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ -){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - - while( *p1 || *p2 ){ - int iCol1; /* The current column index in pp1 */ - int iCol2; /* The current column index in pp2 */ - - if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; - else iCol1 = 0; - - if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; - else iCol2 = 0; - - if( iCol1==iCol2 ){ - sqlite3_int64 i1 = 0; /* Last position from pp1 */ - sqlite3_int64 i2 = 0; /* Last position from pp2 */ - sqlite3_int64 iPrev = 0; - int n = fts3PutColNumber(&p, iCol1); - p1 += n; - p2 += n; - - /* At this point, both p1 and p2 point to the start of column-lists - ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each - ** incremented by 2 before being stored. Each list is terminated by a - ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists - ** and writes the results to buffer p. p is left pointing to the byte - ** after the list written. No terminator (POS_END or POS_COLUMN) is - ** written to the output. - */ - fts3GetDeltaVarint(&p1, &i1); - fts3GetDeltaVarint(&p2, &i2); - do { - fts3PutDeltaVarint(&p, &iPrev, (i1d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); } - - *p++ = POS_END; - *pp = p; - *pp1 = p1 + 1; - *pp2 = p2 + 1; + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( fpos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. -** when the *pp1 token appears before the *pp2 token, but not more than nToken -** slots before it. -** -** e.g. nToken==1 searches for adjacent positions. +** The xUpdate method for rtree module virtual tables. */ -static int fts3PoslistPhraseMerge( - char **pp, /* IN/OUT: Preallocated output buffer */ - int nToken, /* Maximum difference in token positions */ - int isSaveLeft, /* Save the left position */ - int isExact, /* If *pp1 is exactly nTokens before *pp2 */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ +static int rtreeUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid ){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - int iCol1 = 0; - int iCol2 = 0; - - /* Never set both isSaveLeft and isExact for the same invocation. */ - assert( isSaveLeft==0 || isExact==0 ); + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ - assert( p!=0 && *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - } - if( *p2==POS_COLUMN ){ - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } + rtreeReference(pRtree); + assert(nData>=1); - while( 1 ){ - if( iCol1==iCol2 ){ - char *pSave = p; - sqlite3_int64 iPrev = 0; - sqlite3_int64 iPos1 = 0; - sqlite3_int64 iPos2 = 0; + cell.iRowid = 0; /* Used only to suppress a compiler warning */ - if( iCol1 ){ - *p++ = POS_COLUMN; - p += sqlite3Fts3PutVarint(p, iCol1); - } + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; - assert( *p1!=POS_END && *p1!=POS_COLUMN ); - assert( *p2!=POS_END && *p2!=POS_COLUMN ); - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + assert( nData<=(pRtree->nDim*2 + 3) ); - while( 1 ){ - if( iPos2==iPos1+nToken - || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) - ){ - sqlite3_int64 iSave; - iSave = isSaveLeft ? iPos1 : iPos2; - fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; - pSave = 0; - assert( p ); - } - if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ - if( (*p2&0xFE)==0 ) break; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - }else{ - if( (*p1&0xFE)==0 ) break; - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; iicell.aCoord[ii+1].f ){ + rc = SQLITE_CONSTRAINT; + goto constraint; } } - - if( pSave ){ - assert( pp && p ); - p = pSave; + }else +#endif + { + for(ii=0; iicell.aCoord[ii+1].i ){ + rc = SQLITE_CONSTRAINT; + goto constraint; + } } - - fts3ColumnlistCopy(0, &p1); - fts3ColumnlistCopy(0, &p2); - assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); - if( 0==*p1 || 0==*p2 ) break; - - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - p2++; - p2 += fts3GetVarint32(p2, &iCol2); } - /* Advance pointer p1 or p2 (whichever corresponds to the smaller of - ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. - */ - else if( iCol1pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + } + bHaveRowid = 1; } } - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - if( *pp==p ){ - return 0; - } - *p++ = 0x00; - *pp = p; - return 1; -} - -/* -** Merge two position-lists as required by the NEAR operator. The argument -** position lists correspond to the left and right phrases of an expression -** like: -** -** "phrase 1" NEAR "phrase number 2" -** -** Position list *pp1 corresponds to the left-hand side of the NEAR -** expression and *pp2 to the right. As usual, the indexes in the position -** lists are the offsets of the last token in each phrase (tokens "1" and "2" -** in the example above). -** -** The output position list - written to *pp - is a copy of *pp2 with those -** entries that are not sufficiently NEAR entries in *pp1 removed. -*/ -static int fts3PoslistNearMerge( - char **pp, /* Output buffer */ - char *aTmp, /* Temporary buffer space */ - int nRight, /* Maximum difference in token positions */ - int nLeft, /* Maximum difference in token positions */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ -){ - char *p1 = *pp1; - char *p2 = *pp2; - - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); } - return res; -} + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; -/* -** An instance of this function is used to merge together the (potentially -** large number of) doclists for each term that matches a prefix query. -** See function fts3TermSelectMerge() for details. -*/ -typedef struct TermSelect TermSelect; -struct TermSelect { - char *aaOutput[16]; /* Malloc'd output buffers */ - int anOutput[16]; /* Size each output buffer in bytes */ -}; + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; -/* -** This function is used to read a single varint from a buffer. Parameter -** pEnd points 1 byte past the end of the buffer. When this function is -** called, if *pp points to pEnd or greater, then the end of the buffer -** has been reached. In this case *pp is set to 0 and the function returns. -** -** If *pp does not point to or past pEnd, then a single varint is read -** from *pp. *pp is then set to point 1 byte past the end of the read varint. -** -** If bDescIdx is false, the value read is added to *pVal before returning. -** If it is true, the value read is subtracted from *pVal before this -** function returns. -*/ -static void fts3GetDeltaVarint3( - char **pp, /* IN/OUT: Point to read varint from */ - char *pEnd, /* End of buffer */ - int bDescIdx, /* True if docids are descending */ - sqlite3_int64 *pVal /* IN/OUT: Integer value */ -){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - if( bDescIdx ){ - *pVal -= iVal; - }else{ - *pVal += iVal; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } } } + +constraint: + rtreeRelease(pRtree); + return rc; } /* -** This function is used to write a single varint to a buffer. The varint -** is written to *pp. Before returning, *pp is set to point 1 byte past the -** end of the value written. -** -** If *pbFirst is zero when this function is called, the value written to -** the buffer is that of parameter iVal. -** -** If *pbFirst is non-zero when this function is called, then the value -** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) -** (if bDescIdx is non-zero). -** -** Before returning, this function always sets *pbFirst to 1 and *piPrev -** to the value of parameter iVal. +** The xRename method for rtree module virtual tables. */ -static void fts3PutDeltaVarint3( - char **pp, /* IN/OUT: Output pointer */ - int bDescIdx, /* True for descending docids */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - int *pbFirst, /* IN/OUT: True after first int written */ - sqlite3_int64 iVal /* Write this value to the list */ -){ - sqlite3_int64 iWrite; - if( bDescIdx==0 || *pbFirst==0 ){ - iWrite = iVal - *piPrev; - }else{ - iWrite = *piPrev - iVal; +static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" + "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" + "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + ); + if( zSql ){ + rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); + sqlite3_free(zSql); } - assert( *pbFirst || *piPrev==0 ); - assert( *pbFirst==0 || iWrite>0 ); - *pp += sqlite3Fts3PutVarint(*pp, iWrite); - *piPrev = iVal; - *pbFirst = 1; + return rc; } - -/* -** This macro is used by various functions that merge doclists. The two -** arguments are 64-bit docid values. If the value of the stack variable -** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). -** Otherwise, (i2-i1). -** -** Using this makes it easier to write code that can merge doclists that are -** sorted in either ascending or descending order. -*/ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) - /* -** This function does an "OR" merge of two doclists (output contains all -** positions contained in either argument doclist). If the docids in the -** input doclists are sorted in ascending order, parameter bDescDoclist -** should be false. If they are sorted in ascending order, it should be -** passed a non-zero value. -** -** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer -** containing the output doclist and SQLITE_OK is returned. In this case -** *pnOut is set to the number of bytes in the output doclist. -** -** If an error occurs, an SQLite error code is returned. The output values -** are undefined in this case. +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. */ -static int fts3DoclistOrMerge( - int bDescDoclist, /* True if arguments are desc */ - char *a1, int n1, /* First doclist */ - char *a2, int n2, /* Second doclist */ - char **paOut, int *pnOut /* OUT: Malloc'd doclist */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - char *p1 = a1; - char *p2 = a2; - char *p; - char *aOut; - int bFirstOut = 0; - - *paOut = 0; - *pnOut = 0; - - /* Allocate space for the output. Both the input and output doclists - ** are delta encoded. If they are in ascending order (bDescDoclist==0), - ** then the first docid in each list is simply encoded as a varint. For - ** each subsequent docid, the varint stored is the difference between the - ** current and previous docid (a positive number - since the list is in - ** ascending order). - ** - ** The first docid written to the output is therefore encoded using the - ** same number of bytes as it is in whichever of the input lists it is - ** read from. And each subsequent docid read from the same input list - ** consumes either the same or less bytes as it did in the input (since - ** the difference between it and the previous value in the output must - ** be a positive value less than or equal to the delta value read from - ** the input list). The same argument applies to all but the first docid - ** read from the 'other' list. And to the contents of all position lists - ** that will be copied and merged from the input to the output. - ** - ** However, if the first docid copied to the output is a negative number, - ** then the encoding of the first docid from the 'other' input list may - ** be larger in the output than it was in the input (since the delta value - ** may be a larger positive integer than the actual docid). - ** - ** The space required to store the output is therefore the sum of the - ** sizes of the two inputs, plus enough space for exactly one of the input - ** docids to grow. - ** - ** A symetric argument may be made if the doclists are in descending - ** order. - */ - aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); - if( !aOut ) return SQLITE_NOMEM; +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; - p = aOut; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - while( p1 || p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } - if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); - fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } } + sqlite3_free(zSql); } - *paOut = aOut; - *pnOut = (int)(p-aOut); - assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); - return SQLITE_OK; + return rc; } -/* -** This function does a "phrase" merge of two doclists. In a phrase merge, -** the output contains a copy of each position from the right-hand input -** doclist for which there is a position in the left-hand input doclist -** exactly nDist tokens before it. -** -** If the docids in the input doclists are sorted in ascending order, -** parameter bDescDoclist should be false. If they are sorted in ascending -** order, it should be passed a non-zero value. -** -** The right-hand input doclist is overwritten by this function. -*/ -static void fts3DoclistPhraseMerge( - int bDescDoclist, /* True if arguments are desc */ - int nDist, /* Distance from left to right (1=adjacent) */ - char *aLeft, int nLeft, /* Left doclist */ - char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[*pnRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - int bFirstOut = 0; - char *aOut = aRight; - - assert( nDist>0 ); +static sqlite3_module rtreeModule = { + 0, /* iVersion */ + rtreeCreate, /* xCreate - create a table */ + rtreeConnect, /* xConnect - connect to an existing table */ + rtreeBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + rtreeFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + rtreeColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + rtreeUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; - p = aOut; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); +static int rtreeSqlInit( + Rtree *pRtree, + sqlite3 *db, + const char *zDb, + const char *zPrefix, + int isCreate +){ + int rc = SQLITE_OK; - while( p1 && p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); - if( iDiff==0 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - int bFirstOutSave = bFirstOut; + #define N_STATEMENT 9 + static const char *azSql[N_STATEMENT] = { + /* Read and write the xxx_node table */ + "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - bFirstOut = bFirstOutSave; - } - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - } - } + /* Read and write the xxx_rowid table */ + "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1", + "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1", - *pnRight = (int)(p - aOut); -} + /* Read and write the xxx_parent table */ + "SELECT parentnode FROM '%q'.'%q_parent' WHERE nodeno = :1", + "INSERT OR REPLACE INTO '%q'.'%q_parent' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_parent' WHERE nodeno = :1" + }; + sqlite3_stmt **appStmt[N_STATEMENT]; + int i; -/* -** Argument pList points to a position list nList bytes in size. This -** function checks to see if the position list contains any entries for -** a token in position 0 (of any column). If so, it writes argument iDelta -** to the output buffer pOut, followed by a position list consisting only -** of the entries from pList at position 0, and terminated by an 0x00 byte. -** The value returned is the number of bytes written to pOut (if any). -*/ -SQLITE_PRIVATE int sqlite3Fts3FirstFilter( - sqlite3_int64 iDelta, /* Varint that may be written to pOut */ - char *pList, /* Position list (no 0x00 term) */ - int nList, /* Size of pList in bytes */ - char *pOut /* Write output here */ -){ - int nOut = 0; - int bWritten = 0; /* True once iDelta has been written */ - char *p = pList; - char *pEnd = &pList[nList]; + pRtree->db = db; - if( *p!=0x01 ){ - if( *p==0x02 ){ - nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); - pOut[nOut++] = 0x02; - bWritten = 1; + if( isCreate ){ + char *zCreate = sqlite3_mprintf( +"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" +"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" +"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", + zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize + ); + if( !zCreate ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc!=SQLITE_OK ){ + return rc; } - fts3ColumnlistCopy(0, &p); } - while( ppReadNode; + appStmt[1] = &pRtree->pWriteNode; + appStmt[2] = &pRtree->pDeleteNode; + appStmt[3] = &pRtree->pReadRowid; + appStmt[4] = &pRtree->pWriteRowid; + appStmt[5] = &pRtree->pDeleteRowid; + appStmt[6] = &pRtree->pReadParent; + appStmt[7] = &pRtree->pWriteParent; + appStmt[8] = &pRtree->pDeleteParent; + + rc = rtreeQueryStat1(db, pRtree); + for(i=0; iaaOutput); i++){ - if( pTS->aaOutput[i] ){ - if( !aOut ){ - aOut = pTS->aaOutput[i]; - nOut = pTS->anOutput[i]; - pTS->aaOutput[i] = 0; - }else{ - int nNew; - char *aNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, - pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - return rc; - } - - sqlite3_free(pTS->aaOutput[i]); - sqlite3_free(aOut); - pTS->aaOutput[i] = 0; - aOut = aNew; - nOut = nNew; +static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ + int rc = SQLITE_NOMEM; + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *piVal = sqlite3_column_int(pStmt, 0); } + rc = sqlite3_finalize(pStmt); } } - - pTS->aaOutput[0] = aOut; - pTS->anOutput[0] = nOut; - return SQLITE_OK; + return rc; } /* -** Merge the doclist aDoclist/nDoclist into the TermSelect object passed -** as the first argument. The merge is an "OR" merge (see function -** fts3DoclistOrMerge() for details). +** This function is called from within the xConnect() or xCreate() method to +** determine the node-size used by the rtree table being created or connected +** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. ** -** This function is called with the doclist for each term that matches -** a queried prefix. It merges all these doclists into one, the doclist -** for the specified prefix. Since there can be a very large number of -** doclists to merge, the merging is done pair-wise using the TermSelect -** object. +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. ** -** This function returns SQLITE_OK if the merge is successful, or an -** SQLite error code (SQLITE_NOMEM) if an error occurs. +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. */ -static int fts3TermSelectMerge( - Fts3Table *p, /* FTS table handle */ - TermSelect *pTS, /* TermSelect object to merge into */ - char *aDoclist, /* Pointer to doclist */ - int nDoclist /* Size of aDoclist in bytes */ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ ){ - if( pTS->aaOutput[0]==0 ){ - /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist); - pTS->anOutput[0] = nDoclist; - if( pTS->aaOutput[0] ){ - memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + int rc; + char *zSql; + if( isCreate ){ + int iPageSize = 0; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } }else{ - return SQLITE_NOMEM; + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } }else{ - char *aMerge = aDoclist; - int nMerge = nDoclist; - int iOut; - - for(iOut=0; iOutaaOutput); iOut++){ - if( pTS->aaOutput[iOut]==0 ){ - assert( iOut>0 ); - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - break; - }else{ - char *aNew; - int nNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, - pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - return rc; - } - - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - } - } + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } } - return SQLITE_OK; -} -/* -** Append SegReader object pNew to the end of the pCsr->apSegment[] array. -*/ -static int fts3SegReaderCursorAppend( - Fts3MultiSegReader *pCsr, - Fts3SegReader *pNew -){ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - pCsr->apSegment = apNew; - } - pCsr->apSegment[pCsr->nSegment++] = pNew; - return SQLITE_OK; + sqlite3_free(zSql); + return rc; } -/* -** Add seg-reader objects to the Fts3MultiSegReader object passed as the -** 8th argument. +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. ** -** This function returns SQLITE_OK if successful, or an SQLite error code -** otherwise. +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... */ -static int fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ +static int rtreeInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ ){ - int rc = SQLITE_OK; /* Error code */ - sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ - int rc2; /* Result of sqlite3_reset() */ + int rc = SQLITE_OK; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); - /* If iLevel is less than 0 and this is not a scan, include a seg-reader - ** for the pending-terms. If this is a scan, then this call must be being - ** made by an fts4aux module, not an FTS table. In this case calling - ** Fts3SegReaderPending might segfault, as the data structures used by - ** fts4aux are not completely populated. So it's easiest to filter these - ** calls out here. */ - if( iLevel<0 && p->aIndex ){ - Fts3SegReader *pSeg = 0; - rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); - if( rc==SQLITE_OK && pSeg ){ - rc = fts3SegReaderCursorAppend(pCsr, pSeg); - } + const char *aErrMsg[] = { + 0, /* 0 */ + "Wrong number of columns for an rtree table", /* 1 */ + "Too few columns for an rtree table", /* 2 */ + "Too many columns for an rtree table" /* 3 */ + }; + + int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; + if( aErrMsg[iErr] ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + return SQLITE_ERROR; } - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); - } + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pSeg = 0; + /* Allocate the sqlite3_vtab structure */ + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->nDim = (argc-4)/2; + pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; + pRtree->eCoordType = eCoordType; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); - /* Read the values returned by the SELECT into local variables. */ - sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); - sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); - sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); - /* If zTerm is not NULL, and this segment is not stored entirely on its - ** root node, the range of leaves scanned can be reduced. Do this. */ - if( iStartBlock && zTerm ){ - sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); - if( rc!=SQLITE_OK ) goto finished; - if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + if( rc==SQLITE_OK ){ + if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else{ + char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); + char *zTmp; + int ii; + for(ii=4; zSql && iinSegment+1, - (isPrefix==0 && isScan==0), - iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg - ); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegReaderCursorAppend(pCsr, pSeg); + if( zSql ){ + zTmp = zSql; + zSql = sqlite3_mprintf("%s);", zTmp); + sqlite3_free(zTmp); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); } } - finished: - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_DONE ) rc = rc2; - + if( rc==SQLITE_OK ){ + *ppVtab = (sqlite3_vtab *)pRtree; + }else{ + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + } return rc; } -/* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language-id to search */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ -){ - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevel*2 coordinates. */ -static int fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ -){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ - int rc = SQLITE_NOMEM; /* Return code */ +static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + char *zText = 0; + RtreeNode node; + Rtree tree; + int ii; - pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); - if( pSegcsr ){ - int i; - int bFound = 0; /* True once an index has been found */ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + UNUSED_PARAMETER(nArg); + memset(&node, 0, sizeof(RtreeNode)); + memset(&tree, 0, sizeof(Rtree)); + tree.nDim = sqlite3_value_int(apArg[0]); + tree.nBytesPerCell = 8 + 8 * tree.nDim; + node.zData = (u8 *)sqlite3_value_blob(apArg[1]); - if( isPrefix ){ - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr - ); - pSegcsr->bLookup = 1; - } - } + for(ii=0; iinIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm+1 ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr - ); - if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero( - p, pCsr->iLangid, zTerm, nTerm, pSegcsr - ); - } - } - } + nodeGetCell(&tree, &node, ii, &cell); + sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); + nCell = (int)strlen(zCell); + for(jj=0; jjiLangid, - 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr - ); - pSegcsr->bLookup = !isPrefix; + if( zText ){ + char *zTextNew = sqlite3_mprintf("%s {%s}", zText, zCell); + sqlite3_free(zText); + zText = zTextNew; + }else{ + zText = sqlite3_mprintf("{%s}", zCell); } } - - *ppSegcsr = pSegcsr; - return rc; + + sqlite3_result_text(ctx, zText, -1, sqlite3_free); } -/* -** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +/* This routine implements an SQL function that returns the "depth" parameter +** from the front of a blob that is an r-tree node. For example: +** +** SELECT rtreedepth(data) FROM rt_node WHERE nodeno=1; +** +** The depth value is 0 for all nodes other than the root node, and the root +** node always has nodeno=1, so the example above is the primary use for this +** routine. This routine is intended for testing and analysis only. */ -static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ - sqlite3Fts3SegReaderFinish(pSegcsr); - sqlite3_free(pSegcsr); +static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + UNUSED_PARAMETER(nArg); + if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB + || sqlite3_value_bytes(apArg[0])<2 + ){ + sqlite3_result_error(ctx, "Invalid argument to rtreedepth()", -1); + }else{ + u8 *zBlob = (u8 *)sqlite3_value_blob(apArg[0]); + sqlite3_result_int(ctx, readInt16(zBlob)); + } } /* -** This function retrieves the doclist for the specified term (or term -** prefix) from the database. +** Register the r-tree module with database handle db. This creates the +** virtual table module "rtree" and the debugging/analysis scalar +** function "rtreenode". */ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - Fts3PhraseToken *pTok, /* Token to query for */ - int iColumn, /* Column to query (or -ve for all columns) */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ -){ - int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Object for pair-wise doclist merging */ - Fts3SegFilter filter; /* Segment term filter configuration */ - - pSegcsr = pTok->pSegcsr; - memset(&tsc, 0, sizeof(TermSelect)); - - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS - | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) - | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); - filter.iCol = iColumn; - filter.zTerm = pTok->z; - filter.nTerm = pTok->n; +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ + const int utf8 = SQLITE_UTF8; + int rc; - rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); - while( SQLITE_OK==rc - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) - ){ - rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); } - if( rc==SQLITE_OK ){ - rc = fts3TermSelectFinishMerge(p, &tsc); +#ifdef SQLITE_RTREE_INT_ONLY + void *c = (void *)RTREE_COORD_INT32; +#else + void *c = (void *)RTREE_COORD_REAL32; +#endif + rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); } if( rc==SQLITE_OK ){ - *ppOut = tsc.aaOutput[0]; - *pnOut = tsc.anOutput[0]; - }else{ - int i; - for(i=0; ipSegcsr = 0; return rc; } /* -** This function counts the total number of docids in the doclist stored -** in buffer aList[], size nList bytes. -** -** If the isPoslist argument is true, then it is assumed that the doclist -** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded -** varints. +** This routine deletes the RtreeGeomCallback object that was attached +** one of the SQL functions create by sqlite3_rtree_geometry_callback() +** or sqlite3_rtree_query_callback(). In other words, this routine is the +** destructor for an RtreeGeomCallback objecct. This routine is called when +** the corresponding SQL function is deleted. */ -static int fts3DoclistCountDocids(char *aList, int nList){ - int nDoc = 0; /* Return value */ - if( aList ){ - char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Cursor */ - while( pxDestructor ) pInfo->xDestructor(pInfo->pContext); + sqlite3_free(p); } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. -** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. +** This routine frees the BLOB that is returned by geomCallback(). */ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int rc; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - }else{ - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - rc = SQLITE_OK; - } - }else{ - rc = fts3EvalNext((Fts3Cursor *)pCursor); +static void rtreeMatchArgFree(void *pArg){ + int i; + RtreeMatchArg *p = (RtreeMatchArg*)pArg; + for(i=0; inParam; i++){ + sqlite3_value_free(p->apSqlParam[i]); } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; + sqlite3_free(p); } /* -** The following are copied from sqliteInt.h. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. ** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + int memErr = 0; -/* -** If the numeric type of argument pVal is "integer", then return it -** converted to a 64-bit signed integer. Otherwise, return a copy of -** the second parameter, iDefault. -*/ -static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ - if( pVal ){ - int eType = sqlite3_value_numeric_type(pVal); - if( eType==SQLITE_INTEGER ){ - return sqlite3_value_int64(pVal); + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) + + nArg*sizeof(sqlite3_value*); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->cb = pGeomCtx[0]; + pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; + pBlob->nParam = nArg; + for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); + if( pBlob->apSqlParam[i]==0 ) memErr = 1; +#ifdef SQLITE_RTREE_INT_ONLY + pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#else + pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif + } + if( memErr ){ + sqlite3_result_error_nomem(ctx); + rtreeMatchArgFree(pBlob); + }else{ + sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); } } - return iDefault; } /* -** This is the xFilter interface for the virtual table. See -** the virtual table xFilter method documentation for additional -** information. -** -** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against -** the %_content table. -** -** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry -** in the %_content table. -** -** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The -** column on the left-hand side of the MATCH operator is column -** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand -** side of the MATCH operator. +** Register a new geometry function for use with the r-tree MATCH operator. */ -static int fts3FilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ - int rc; - char *zSql; /* SQL statement used to access %_content */ - int eSearch; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ - sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ - sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ - sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ - int iIdx; - - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); - - eSearch = (idxNum & 0x0000FFFF); - assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( p->pSegments==0 ); - - /* Collect arguments into local variables */ - iIdx = 0; - if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; - assert( iIdx==nVal ); - - /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - - /* Set the lower and upper bounds on docids to return */ - pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); - pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); - - if( idxStr ){ - pCsr->bDesc = (idxStr[0]=='D'); - }else{ - pCsr->bDesc = p->bDescIdx; - } - pCsr->eSearch = (i16)eSearch; - - if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ - int iCol = eSearch-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(pCons); - - if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ - return SQLITE_NOMEM; - } - - pCsr->iLangid = 0; - if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); - - assert( p->base.zErrMsg==0 ); - rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, - &p->base.zErrMsg - ); - if( rc!=SQLITE_OK ){ - return rc; - } - - rc = fts3EvalStart(pCsr); - sqlite3Fts3SegmentsClose(p); - if( rc!=SQLITE_OK ) return rc; - pCsr->pNextId = pCsr->aDoclist; - pCsr->iPrevId = 0; - } - - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - if( eSearch==FTS3_FULLSCAN_SEARCH ){ - zSql = sqlite3_mprintf( - "SELECT %s ORDER BY rowid %s", - p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") - ); - if( zSql ){ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); - } - } - if( rc!=SQLITE_OK ) return rc; + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ - return fts3NextMethod(pCursor); + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); } -/* -** This is the xEof method of the virtual table. SQLite calls this -** routine to find out if it has reached the end of a result set. +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. */ -static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - return ((Fts3Cursor *)pCursor)->isEof; +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); } -/* -** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. -*/ -static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - *pRowid = pCsr->iPrevId; - return SQLITE_OK; +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3RtreeInit(db); } +#endif -/* -** This is the xColumn method, called by SQLite to request a value from -** the row that the supplied cursor currently points to. +#endif + +/************** End of rtree.c ***********************************************/ +/************** Begin file icu.c *********************************************/ +/* +** 2007 May 6 ** -** If: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** (iCol < p->nColumn) -> The value of the iCol'th user column. -** (iCol == p->nColumn) -> Magic column with the same name as the table. -** (iCol == p->nColumn+1) -> Docid column -** (iCol == p->nColumn+2) -> Langid column +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. */ -static int fts3ColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - - /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+2 ); - - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. */ - sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - }else{ - /* The requested column is either a user column (one that contains - ** indexed data), or the language-id column. */ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+2 ){ - int iLangid = 0; - if( p->zLanguageid ){ - iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); - } - sqlite3_result_int(pCtx, iLangid); - }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); - } - } - } +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; -} +/* Include ICU headers */ +#include +#include +#include +#include -/* -** This function is the implementation of the xUpdate callback used by -** FTS3 virtual tables. It is invoked by SQLite each time a row is to be -** inserted, updated or deleted. +/* #include */ + +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. */ -static int fts3UpdateMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); } /* -** Implementation of xSync() method. Flush the contents of the pending-terms -** hash-table to the database. +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. */ -static int fts3SyncMethod(sqlite3_vtab *pVtab){ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const int MATCH_ONE = (UChar32)'_'; + static const int MATCH_ALL = (UChar32)'%'; - /* Following an incremental-merge operation, assuming that the input - ** segments are not completely consumed (the usual case), they are updated - ** in place to remove the entries that have already been merged. This - ** involves updating the leaf block that contains the smallest unmerged - ** entry and each block (if any) between the leaf and the root node. So - ** if the height of the input segment b-trees is N, and input segments - ** are merged eight at a time, updating the input segments at the end - ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually - ** small - often between 0 and 2. So the overhead of the incremental - ** merge is somewhere between 8 and 24 blocks. To avoid this overhead - ** dwarfing the actual productive work accomplished, the incremental merge - ** is only attempted if it will write at least 64 leaf blocks. Hence - ** nMinMerge. - ** - ** Of course, updating the input segments also involves deleting a bunch - ** of blocks from the segments table. But this is not considered overhead - ** as it would also be required by a crisis-merge that used the same input - ** segments. - */ - const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + int iPattern = 0; /* Current byte index in zPattern */ + int iString = 0; /* Current byte index in zString */ - Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + int prevEscape = 0; /* True if the previous character was uEsc */ - if( rc==SQLITE_OK - && p->nLeafAdd>(nMinMerge/16) - && p->nAutoincrmerge && p->nAutoincrmerge!=0xff - ){ - int mxLevel = 0; /* Maximum relative level value in db */ - int A; /* Incr-merge parameter A */ + while( zPattern[iPattern]!=0 ){ - rc = sqlite3Fts3MaxLevel(p, &mxLevel); - assert( rc==SQLITE_OK || mxLevel==0 ); - A = p->nLeafAdd * mxLevel; - A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); - } - sqlite3Fts3SegmentsClose(p); - return rc; -} + /* Read (and consume) the next character from the input pattern. */ + UChar32 uPattern; + U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); -/* -** If it is currently unknown whether or not the FTS table has an %_stat -** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat -** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. -*/ -static int fts3SetHasStat(Fts3Table *p){ - int rc = SQLITE_OK; - if( p->bHasStat==2 ){ - const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; - char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); - if( zSql ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( !prevEscape && uPattern==MATCH_ALL ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); + } + iPattern++; } - sqlite3_free(zSql); + + if( zPattern[iPattern]==0 ) return 1; + + while( zString[iString] ){ + if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + return 1; + } + U8_FWD_1_UNSAFE(zString, iString); + } + return 0; + + }else if( !prevEscape && uPattern==MATCH_ONE ){ + /* Case 2. */ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); + + }else if( !prevEscape && uPattern==uEsc){ + /* Case 3. */ + prevEscape = 1; + }else{ - rc = SQLITE_NOMEM; + /* Case 4. */ + UChar32 uString; + U8_NEXT_UNSAFE(zString, iString, uString); + uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); + uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; } } - return rc; -} -/* -** Implementation of xBegin() method. -*/ -static int fts3BeginMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(pVtab); - assert( p->pSegments==0 ); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=1 ); - TESTONLY( p->inTransaction = 1 ); - TESTONLY( p->mxSavepoint = -1; ); - p->nLeafAdd = 0; - return fts3SetHasStat(p); + return zString[iString]==0; } /* -** Implementation of xCommit() method. This is a no-op. The contents of -** the pending-terms hash-table have already been flushed into the database -** by fts3SyncMethod(). +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). */ -static int fts3CommitMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(pVtab); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=0 ); - assert( p->pSegments==0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; -} +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; -/* -** Implementation of xRollback(). Discard the contents of the pending-terms -** hash-table. Any changes made to the database are reverted by SQLite. -*/ -static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - sqlite3Fts3PendingTermsClear(p); - assert( p->inTransaction!=0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; -} + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } -/* -** When called, *ppPoslist must point to the byte immediately following the -** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function -** moves *ppPoslist so that it instead points to the first byte of the -** same position list. -*/ -static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-2]; - char c = 0; - while( p>pStart && (c=*p--)==0 ); - while( p>pStart && (*p & 0x80) | c ){ - c = *p--; + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); } - if( p>pStart ){ p = &p[2]; } - while( *p++&0x80 ); - *ppPoslist = p; } /* -** Helper function used by the implementation of the overloaded snippet(), -** offsets() and optimize() SQL functions. +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. ** -** If the value passed as the third argument is a blob of size -** sizeof(Fts3Cursor*), then the blob contents are copied to the -** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error -** message is written to context pContext and SQLITE_ERROR returned. The -** string passed via zFunc is used as part of the error message. +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. */ -static int fts3FunctionArg( - sqlite3_context *pContext, /* SQL function call context */ - const char *zFunc, /* Function name */ - sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ ){ - Fts3Cursor *pRet; - if( sqlite3_value_type(pVal)!=SQLITE_BLOB - || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) - ){ - char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - return SQLITE_ERROR; - } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); } /* -** Implementation of the snippet() function for FTS3 +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). */ -static void fts3SnippetFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of apVal[] array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - int iCol = -1; - int nToken = 15; /* Default number of tokens in snippet */ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} - /* There must be at least one argument passed to this function (otherwise - ** the non-overloaded version would have been called instead of this one). - */ - assert( nVal>=1 ); +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); - if( nVal>6 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function snippet()", -1); + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ return; } - if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; - switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); - case 5: iCol = sqlite3_value_int(apVal[4]); - case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); - case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); - case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + }else{ + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; + } } - if( !zEllipsis || !zEnd || !zStart ){ - sqlite3_result_error_nomem(pContext); - }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; } -} -/* -** Implementation of the offsets() function for FTS3 -*/ -static void fts3OffsetsFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } - UNUSED_PARAMETER(nVal); + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; - assert( pCsr ); - if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Offsets(pContext, pCsr); - } + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); } -/* -** Implementation of the special optimize() function for FTS3. This -** function merges all segments in the database to a single segment. -** Example usage is: +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). ** -** SELECT optimize(t) FROM t LIMIT 1; +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. ** -** where 't' is the name of an FTS3 table. +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings */ -static void fts3OptimizeFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - int rc; /* Return code */ - Fts3Table *p; /* Virtual table handle */ - Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; + UChar *zOutput; + int nInput; + int nOutput; - UNUSED_PARAMETER(nVal); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale = 0; - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; - p = (Fts3Table *)pCursor->base.pVtab; - assert( p ); + assert(nArg==1 || nArg==2); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } - rc = sqlite3Fts3Optimize(p); + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nInput = sqlite3_value_bytes16(apArg[0]); - switch( rc ){ - case SQLITE_OK: - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - break; - case SQLITE_DONE: - sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); - break; - default: - sqlite3_result_error_code(pContext, rc); - break; + nOutput = nInput * 2 + 2; + zOutput = sqlite3_malloc(nOutput); + if( !zOutput ){ + return; + } + + if( sqlite3_user_data(p) ){ + u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); + }else{ + u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); + } + + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "u_strToLower()/u_strToUpper", status); + return; } + + sqlite3_result_text16(p, zOutput, -1, xFree); } /* -** Implementation of the matchinfo() function for FTS3 +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). */ -static void fts3MatchinfoFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - assert( nVal==1 || nVal==2 ); - if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = 0; - if( nVal>1 ){ - zArg = (const char *)sqlite3_value_text(apVal[1]); - } - sqlite3Fts3Matchinfo(pContext, pCsr, zArg); - } +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); } /* -** This routine implements the xFindFunction method for the FTS3 -** virtual table. +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). */ -static int fts3FindFunctionMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* Unused */ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight ){ - struct Overloaded { - const char *zName; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aOverload[] = { - { "snippet", fts3SnippetFunc }, - { "offsets", fts3OffsetsFunc }, - { "optimize", fts3OptimizeFunc }, - { "matchinfo", fts3MatchinfoFunc }, - }; - int i; /* Iterator variable */ - - UNUSED_PARAMETER(pVtab); - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(ppArg); - - for(i=0; i, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. */ -static int fts3RenameMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - const char *zName /* New name of table */ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg ){ - Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; /* Database connection */ - int rc; /* Return Code */ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - rc = fts3SetHasStat(p); - - /* As it happens, the pending terms table is always empty here. This is - ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction - ** always opens a savepoint transaction. And the xSavepoint() method - ** flushes the pending terms table. But leave the (no-op) call to - ** PendingTermsFlush() in in case that changes. - */ - assert( p->nPendingData==0 ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); - } + assert(nArg==2); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); + if( !zLocale || !zName ){ + return; } - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", - p->zDb, p->zName, zName - ); - } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", - p->zDb, p->zName, zName - ); + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", - p->zDb, p->zName, zName - ); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", - p->zDb, p->zName, zName + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + icuCollationColl, icuCollationDel ); - return rc; + if( rc!=SQLITE_OK ){ + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } } /* -** The xSavepoint() method. -** -** Flush the contents of the pending-terms table to disk. +** Register the ICU extension functions with database db. */ -static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ + struct IcuScalar { + const char *zName; /* Function name */ + int nArg; /* Number of arguments */ + int enc; /* Optimal text encoding */ + void *pContext; /* sqlite3_user_data() context */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } scalars[] = { + {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, + + {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + + {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + + {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, + + {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + }; + int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + int i; + + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ + struct IcuScalar *p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + ); } - return rc; -} -/* -** The xRelease() method. -** -** This is a no-op. -*/ -static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); - return SQLITE_OK; + return rc; } -/* -** The xRollbackTo() method. -** -** Discard the contents of the pending terms table. -*/ -static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); - return SQLITE_OK; +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); } +#endif -static const sqlite3_module fts3Module = { - /* iVersion */ 2, - /* xCreate */ fts3CreateMethod, - /* xConnect */ fts3ConnectMethod, - /* xBestIndex */ fts3BestIndexMethod, - /* xDisconnect */ fts3DisconnectMethod, - /* xDestroy */ fts3DestroyMethod, - /* xOpen */ fts3OpenMethod, - /* xClose */ fts3CloseMethod, - /* xFilter */ fts3FilterMethod, - /* xNext */ fts3NextMethod, - /* xEof */ fts3EofMethod, - /* xColumn */ fts3ColumnMethod, - /* xRowid */ fts3RowidMethod, - /* xUpdate */ fts3UpdateMethod, - /* xBegin */ fts3BeginMethod, - /* xSync */ fts3SyncMethod, - /* xCommit */ fts3CommitMethod, - /* xRollback */ fts3RollbackMethod, - /* xFindFunction */ fts3FindFunctionMethod, - /* xRename */ fts3RenameMethod, - /* xSavepoint */ fts3SavepointMethod, - /* xRelease */ fts3ReleaseMethod, - /* xRollbackTo */ fts3RollbackToMethod, -}; - -/* -** This function is registered as the module destructor (called when an -** FTS3 enabled database connection is closed). It frees the memory -** allocated for the tokenizer hash table. -*/ -static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); -} +#endif +/************** End of icu.c *************************************************/ +/************** Begin file fts3_icu.c ****************************************/ /* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are -** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c -** respectively. The following three forward declarations are for functions -** declared in these files used to retrieve the respective implementations. +** 2007 June 22 ** -** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point to the "simple" tokenizer implementation. -** And so on. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. */ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); -#endif +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#endif -/* -** Initialize the fts3 extension. If this extension is built as part -** of the sqlite library, then this function is called directly by -** SQLite. If fts3 is built as a dynamically loadable extension, this -** function is called by the sqlite3_extension_init() entry point. -*/ -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ - int rc = SQLITE_OK; - Fts3Hash *pHash = 0; - const sqlite3_tokenizer_module *pSimple = 0; - const sqlite3_tokenizer_module *pPorter = 0; -#ifndef SQLITE_DISABLE_FTS3_UNICODE - const sqlite3_tokenizer_module *pUnicode = 0; -#endif +/* #include */ +/* #include */ +/* #include "fts3_tokenizer.h" */ -#ifdef SQLITE_ENABLE_ICU - const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif +#include +/* #include */ +/* #include */ +#include -#ifndef SQLITE_DISABLE_FTS3_UNICODE - sqlite3Fts3UnicodeTokenizer(&pUnicode); -#endif +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; -#ifdef SQLITE_TEST - rc = sqlite3Fts3InitTerm(db); - if( rc!=SQLITE_OK ) return rc; -#endif +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; - rc = sqlite3Fts3InitAux(db); - if( rc!=SQLITE_OK ) return rc; +struct IcuCursor { + sqlite3_tokenizer_cursor base; - sqlite3Fts3SimpleTokenizerModule(&pSimple); - sqlite3Fts3PorterTokenizerModule(&pPorter); + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ - /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); - if( !pHash ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); - } + int nBuffer; + char *zBuffer; - /* Load the built-in tokenizers into the hash table */ - if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + int iToken; +}; -#ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) -#endif -#ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) -#endif - ){ - rc = SQLITE_NOMEM; - } - } +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; -#ifdef SQLITE_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db); + if( argc>0 ){ + n = strlen(argv[0])+1; } -#endif + p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); - /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the - ** module with sqlite. - */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) - ){ - rc = sqlite3_create_module_v2( - db, "fts3", &fts3Module, (void *)pHash, hashDestroy - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 - ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); - } - return rc; + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); } + *ppTokenizer = (sqlite3_tokenizer *)p; - /* An error has occurred. Delete the hash table and return the error code. */ - assert( rc!=SQLITE_OK ); - if( pHash ){ - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); - } - return rc; + return SQLITE_OK; } /* -** Allocate an Fts3MultiSegReader for each token in the expression headed -** by pExpr. -** -** An Fts3SegReader object is a cursor that can seek or scan a range of -** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple -** Fts3SegReader objects internally to provide an interface to seek or scan -** within the union of all segments of a b-tree. Hence the name. -** -** If the allocated Fts3MultiSegReader just seeks to a single entry in a -** segment b-tree (if the term is not a prefix or it is a prefix for which -** there exists prefix b-tree of the right length) then it may be traversed -** and merged incrementally. Otherwise, it has to be merged into an in-memory -** doclist and then traversed. +** Destroy a tokenizer */ -static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Allocate readers for this expression */ - int *pnToken, /* OUT: Total number of tokens in phrase. */ - int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; - int nToken = pExpr->pPhrase->nToken; - *pnToken += nToken; - for(i=0; ipPhrase->aToken[i]; - int rc = fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - if( rc!=SQLITE_OK ){ - *pRc = rc; - return; - } - } - assert( pExpr->pPhrase->iDoclistToken==0 ); - pExpr->pPhrase->iDoclistToken = -1; - }else{ - *pnOr += (pExpr->eType==FTSQUERY_OR); - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); - } - } +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; } /* -** Arguments pList/nList contain the doclist for token iToken of phrase p. -** It is merged into the main doclist stored in p->doclist.aAll/nAll. -** -** This function assumes that pList points to a buffer allocated using -** sqlite3_malloc(). This function takes responsibility for eventually -** freeing the buffer. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -static void fts3EvalPhraseMergeToken( - Fts3Table *pTab, /* FTS Table pointer */ - Fts3Phrase *p, /* Phrase to merge pList/nList into */ - int iToken, /* Token pList/nList corresponds to */ - char *pList, /* Pointer to doclist */ - int nList /* Number of bytes in pList */ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ ){ - assert( iToken!=p->iDoclistToken ); + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; - if( pList==0 ){ - sqlite3_free(p->doclist.aAll); - p->doclist.aAll = 0; - p->doclist.nAll = 0; - } + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; - else if( p->iDoclistToken<0 ){ - p->doclist.aAll = pList; - p->doclist.nAll = nList; + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); } - - else if( p->doclist.aAll==0 ){ - sqlite3_free(pList); + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; - else { - char *pLeft; - char *pRight; - int nLeft; - int nRight; - int nDiff; + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; - if( p->iDoclistTokendoclist.aAll; - nLeft = p->doclist.nAll; - pRight = pList; - nRight = nList; - nDiff = iToken - p->iDoclistToken; + if( iInputdoclist.aAll; - nRight = p->doclist.nAll; - pLeft = pList; - nLeft = nList; - nDiff = p->iDoclistToken - iToken; + c = 0; } + } - fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); - sqlite3_free(pLeft); - p->doclist.aAll = pRight; - p->doclist.nAll = nRight; + pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; } + pCsr->nChar = iOut; - if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; } /* -** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist -** does not take deferred tokens into account. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Close a tokenization cursor previously opened by a call to icuOpen(). */ -static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p /* Phrase object */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int iToken; - int rc = SQLITE_OK; - - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &p->aToken[iToken]; - assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - - if( pToken->pSegcsr ){ - int nThis = 0; - char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); - if( rc==SQLITE_OK ){ - fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); - } - } - assert( pToken->pSegcsr==0 ); - } - - return rc; +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** This function is called on each phrase after the position lists for -** any deferred tokens have been loaded into memory. It updates the phrases -** current position list to include only those positions that are really -** instances of the phrase (after considering deferred tokens). If this -** means that the phrase does not appear in the current row, doclist.pList -** and doclist.nList are both zeroed. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Extract the next token from a tokenization cursor. */ -static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; /* Used to iterate through phrase tokens */ - char *aPoslist = 0; /* Position list for deferred tokens */ - int nPoslist = 0; /* Number of bytes in aPoslist */ - int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); - - for(iToken=0; iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - Fts3DeferredToken *pDeferred = pToken->pDeferred; +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; - if( pDeferred ){ - char *pList; - int nList; - int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); - if( rc!=SQLITE_OK ) return rc; + int iStart = 0; + int iEnd = 0; + int nByte = 0; - if( pList==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; + while( iStart==iEnd ){ + UChar32 c; - }else if( aPoslist==0 ){ - aPoslist = pList; - nPoslist = nList; + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; }else{ - char *aOut = pList; - char *p1 = aPoslist; - char *p2 = aOut; - - assert( iPrev>=0 ); - fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); - sqlite3_free(aPoslist); - aPoslist = pList; - nPoslist = (int)(aOut - aPoslist); - if( nPoslist==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; - } + break; } - iPrev = iToken; } + assert(iStart<=iEnd); } - if( iPrev>=0 ){ - int nMaxUndeferred = pPhrase->iDoclistToken; - if( nMaxUndeferred<0 ){ - pPhrase->doclist.pList = aPoslist; - pPhrase->doclist.nList = nPoslist; - pPhrase->doclist.iDocid = pCsr->iPrevId; - pPhrase->doclist.bFreeList = 1; - }else{ - int nDistance; - char *p1; - char *p2; - char *aOut; - - if( nMaxUndeferred>iPrev ){ - p1 = aPoslist; - p2 = pPhrase->doclist.pList; - nDistance = nMaxUndeferred - iPrev; - }else{ - p1 = pPhrase->doclist.pList; - p2 = aPoslist; - nDistance = iPrev - nMaxUndeferred; - } - - aOut = (char *)sqlite3_malloc(nPoslist+8); - if( !aOut ){ - sqlite3_free(aPoslist); + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ return SQLITE_NOMEM; } - - pPhrase->doclist.pList = aOut; - if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ - pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); - }else{ - sqlite3_free(aOut); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - } - sqlite3_free(aPoslist); + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; } - } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; return SQLITE_OK; } /* -** Maximum number of tokens a phrase may have to be considered for the -** incremental doclists strategy. +** The set of routines that implement the simple tokenizer */ -#define MAX_INCR_PHRASE_TOKENS 4 +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; /* -** This function is called for each Fts3Phrase in a full-text query -** expression to initialize the mechanism for returning rows. Once this -** function has been called successfully on an Fts3Phrase, it may be -** used with fts3EvalPhraseNext() to iterate through the matching docids. +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_icu.c ********************************************/ +/************** Begin file sqlite3rbu.c **************************************/ +/* +** 2014 August 30 ** -** If parameter bOptOk is true, then the phrase may (or may not) use the -** incremental loading strategy. Otherwise, the entire doclist is loaded into -** memory within this call. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** */ -static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Error code */ - int i; - - /* Determine if doclists may be loaded from disk incrementally. This is - ** possible if the bOptOk argument is true, the FTS doclists will be - ** scanned in forward order, and the phrase consists of - ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" - ** tokens or prefix tokens that cannot use a prefix-index. */ - int bHaveIncr = 0; - int bIncrOk = (bOptOk - && pCsr->bDesc==pTab->bDescIdx - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 -#ifdef SQLITE_TEST - && pTab->bNoIncrDoclist==0 -#endif - ); - for(i=0; bIncrOk==1 && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ - bIncrOk = 0; - } - if( pToken->pSegcsr ) bHaveIncr = 1; - } - if( bIncrOk && bHaveIncr ){ - /* Use the incremental approach. */ - int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - for(i=0; rc==SQLITE_OK && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; - if( pSegcsr ){ - rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); - } - } - p->bIncr = 1; - }else{ - /* Load the full doclist for the phrase into memory. */ - rc = fts3EvalPhraseLoad(pCsr, p); - p->bIncr = 0; - } +/* #include */ +/* #include */ +/* #include */ - assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); - return rc; -} +/* #include "sqlite3.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/ +/************** Begin file sqlite3rbu.h **************************************/ /* -** This function is used to iterate backwards (from the end to start) -** through doclists. It is used by this module to iterate through phrase -** doclists in reverse and by the fts3_write.c module to iterate through -** pending-terms lists when writing to databases with "order=desc". +** 2014 August 30 ** -** The doclist may be sorted in ascending (parameter bDescIdx==0) or -** descending (parameter bDescIdx==1) order of docid. Regardless, this -** function iterates from the end of the doclist to the beginning. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface for the RBU extension. */ -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( - int bDescIdx, /* True if the doclist is desc */ - char *aDoclist, /* Pointer to entire doclist */ - int nDoclist, /* Length of aDoclist in bytes */ - char **ppIter, /* IN/OUT: Iterator pointer */ - sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* OUT: List length pointer */ - u8 *pbEof /* OUT: End-of-file flag */ -){ - char *p = *ppIter; - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); +/* +** SUMMARY +** +** Writing a transaction containing a large number of operations on +** b-tree indexes that are collectively larger than the available cache +** memory can be very inefficient. +** +** The problem is that in order to update a b-tree, the leaf page (at least) +** containing the entry being inserted or deleted must be modified. If the +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction +** may be loaded from or written to the persistent media multiple times. +** Additionally, because the index updates are likely to be applied in +** random order, access to pages within the database is also likely to be in +** random order, which is itself quite inefficient. +** +** One way to improve the situation is to sort the operations on each index +** by index key before applying them to the b-tree. This leads to an IO +** pattern that resembles a single linear scan through the index b-tree, +** and all but guarantees each modified leaf page is loaded and stored +** exactly once. SQLite uses this trick to improve the performance of +** CREATE INDEX commands. This extension allows it to be used to improve +** the performance of large transactions on existing databases. +** +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process will run for long enough to apply +** the entire update, for example because the update is being applied on a +** mobile device that is frequently rebooted. Even after the writer process +** has committed one or more sub-transactions, other database clients continue +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. +** +** "RBU" stands for "Resumable Bulk Update". As in a large database update +** transmitted via a wireless network to a mobile device. A transaction +** applied using this extension is hence refered to as an "RBU update". +** +** +** LIMITATIONS +** +** An "RBU update" transaction is subject to the following limitations: +** +** * The transaction must consist of INSERT, UPDATE and DELETE operations +** only. +** +** * INSERT statements may not use any default values. +** +** * UPDATE and DELETE statements must identify their target rows by +** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY +** KEY fields may not be updated or deleted. If the table being written +** has no PRIMARY KEY, affected rows must be identified by rowid. +** +** * UPDATE statements may not modify PRIMARY KEY columns. +** +** * No triggers will be fired. +** +** * No foreign key violations are detected or reported. +** +** * CHECK constraints are not enforced. +** +** * No constraint handling mode except for "OR ROLLBACK" is supported. +** +** +** PREPARATION +** +** An "RBU update" is stored as a separate SQLite database. A database +** containing an RBU update is an "RBU database". For each table in the +** target database to be updated, the RBU database should contain a table +** named "data_" containing the same set of columns as the +** target table, and one more - "rbu_control". The data_% table should +** have no PRIMARY KEY or UNIQUE constraints, but each column should have +** the same type as the corresponding column in the target database. +** The "rbu_control" column should have no type at all. For example, if +** the target database contains: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); +** +** Then the RBU database should contain: +** +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); +** +** The order of the columns in the data_% table does not matter. +** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_ naming scheme. +** +** Instead of the plain data_ naming scheme, RBU database tables +** may also be named data_, where is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the the portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** +** If the target database table is a virtual table or a table that has no +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "rbu_rowid". This column is mapped to the tables implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using RBU. For +** example, if the target db contains either of the following: +** +** CREATE VIRTUAL TABLE x1 USING fts3(a, b); +** CREATE TABLE x1(a, b) +** +** then the RBU database should contain: +** +** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control); +** +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the +** target table must be present in the input table. For virtual tables, +** hidden columns are optional - they are updated by RBU if present in +** the input table, or not otherwise. For example, to write to an fts4 +** table with a hidden languageid column such as: +** +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); +** +** Either of the following input table schemas may be used: +** +** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); +** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); +** +** For each row to INSERT into the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 0. The +** other columns should be set to the values that make up the new record +** to insert. +** +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to +** do so results in an SQLITE_MISMATCH error. +** +** For each row to DELETE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 1. The +** real primary key values of the row to delete should be stored in the +** corresponding columns of the data_% table. The values stored in the +** other columns are not used. +** +** For each row to UPDATE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain a value of type text. +** The real primary key values identifying the row to update should be +** stored in the corresponding columns of the data_% table row, as should +** the new values of all columns being update. The text value in the +** "rbu_control" column must contain the same number of characters as +** there are columns in the target database table, and must consist entirely +** of 'x' and '.' characters (or in some special cases 'd' - see below). For +** each column that is being updated, the corresponding character is set to +** 'x'. For those that remain as they are, the corresponding character of the +** rbu_control value should be set to '.'. For example, given the tables +** above, the update statement: +** +** UPDATE t1 SET c = 'usa' WHERE a = 4; +** +** is represented by the data_t1 row created by: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x'); +** +** Instead of an 'x' character, characters of the rbu_control value specified +** for UPDATEs may also be set to 'd'. In this case, instead of updating the +** target table with the value stored in the corresponding data_% column, the +** user-defined SQL function "rbu_delta()" is invoked and the result stored in +** the target table column. rbu_delta() is invoked with two arguments - the +** original value currently stored in the target table column and the +** value specified in the data_xxx table. +** +** For example, this row: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); +** +** is similar to an UPDATE statement such as: +** +** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; +** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** +** If the target database table is a virtual table or a table with no PRIMARY +** KEY, the rbu_control value should not include a character corresponding +** to the rbu_rowid value. For example, this: +** +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) +** VALUES(NULL, 'usa', 12, '.x'); +** +** causes a result similar to: +** +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; +** +** The data_xxx tables themselves should have no PRIMARY KEY declarations. +** However, RBU is more efficient if reading the rows in from each data_xxx +** table in "rowid" order is roughly the same as reading them sorted by +** the PRIMARY KEY of the corresponding target database table. In other +** words, rows should be sorted using the destination table PRIMARY KEY +** fields before they are inserted into the data_xxx tables. +** +** USAGE +** +** The API declared below allows an application to apply an RBU update +** stored on disk to an existing target database. Essentially, the +** application: +** +** 1) Opens an RBU handle using the sqlite3rbu_open() function. +** +** 2) Registers any required virtual table modules with the database +** handle returned by sqlite3rbu_db(). Also, if required, register +** the rbu_delta() implementation. +** +** 3) Calls the sqlite3rbu_step() function one or more times on +** the new handle. Each call to sqlite3rbu_step() performs a single +** b-tree operation, so thousands of calls may be required to apply +** a complete update. +** +** 4) Calls sqlite3rbu_close() to close the RBU update handle. If +** sqlite3rbu_step() has been called enough times to completely +** apply the update to the target database, then the RBU database +** is marked as fully applied. Otherwise, the state of the RBU +** update application is saved in the RBU database for later +** resumption. +** +** See comments below for more detail on APIs. +** +** If an update is only partially applied to the target database by the +** time sqlite3rbu_close() is called, various state information is saved +** within the RBU database. This allows subsequent processes to automatically +** resume the RBU update from where it left off. +** +** To remove all RBU extension state information, returning an RBU database +** to its original contents, it is sufficient to drop all tables that begin +** with the prefix "rbu_" +** +** DATABASE LOCKING +** +** An RBU update may not be applied to a database in WAL mode. Attempting +** to do so is an error (SQLITE_ERROR). +** +** While an RBU handle is open, a SHARED lock may be held on the target +** database file. This means it is possible for other clients to read the +** database, but not to write it. +** +** If an RBU update is started and then suspended before it is completed, +** then an external client writes to the database, then attempting to resume +** the suspended RBU update is also an error (SQLITE_BUSY). +*/ - if( p==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext = 0; - char *pDocid = aDoclist; - char *pEnd = &aDoclist[nDoclist]; - int iMul = 1; +#ifndef _SQLITE3RBU_H +#define _SQLITE3RBU_H - while( pDocid0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); - - if( p==0 ){ - p = aDoclist; - p += sqlite3Fts3GetVarint(p, piDocid); - }else{ - fts3PoslistCopy(0, &p); - if( p>=&aDoclist[nDoclist] ){ - *pbEof = 1; - }else{ - sqlite3_int64 iVar; - p += sqlite3Fts3GetVarint(p, &iVar); - *piDocid += ((bDescIdx ? -1 : 1) * iVar); - } - } - - *ppIter = p; -} +SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +); /* -** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof -** to true if EOF is reached. +** Internally, each RBU connection uses a separate SQLite database +** connection to access the target and rbu update databases. This +** API allows the application direct access to these database handles. +** +** The first argument passed to this function must be a valid, open, RBU +** handle. The second argument should be passed zero to access the target +** database handle, or non-zero to access the rbu update database handle. +** Accessing the underlying database handles may be useful in the +** following scenarios: +** +** * If any target tables are virtual tables, it may be necessary to +** call sqlite3_create_module() on the target database handle to +** register the required virtual table implementations. +** +** * If the data_xxx tables in the RBU source database are virtual +** tables, the application may need to call sqlite3_create_module() on +** the rbu update db handle to any required virtual table +** implementations. +** +** * If the application uses the "rbu_delta()" feature described above, +** it must use sqlite3_create_function() or similar to register the +** rbu_delta() implementation with the target database handle. +** +** If an error has occurred, either while opening or stepping the RBU object, +** this function may return NULL. The error code and message may be collected +** when sqlite3rbu_close() is called. */ -static void fts3EvalDlPhraseNext( - Fts3Table *pTab, - Fts3Doclist *pDL, - u8 *pbEof -){ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } - - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; - } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (int)(pIter - pDL->pList); - - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim(). */ - while( pIterpNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; - } -} +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu); /* -** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +** Do some work towards applying the RBU update to the target db. +** +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the RBU update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3rbu_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops +** that immediately return the same value. */ -typedef struct TokenDoclist TokenDoclist; -struct TokenDoclist { - int bIgnore; - sqlite3_int64 iDocid; - char *pList; - int nList; -}; +SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *pRbu); /* -** Token pToken is an incrementally loaded token that is part of a -** multi-token phrase. Advance it to the next matching document in the -** database and populate output variable *p with the details of the new -** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** Force RBU to save its state to disk. ** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int incrPhraseTokenNext( - Fts3Table *pTab, /* Virtual table handle */ - Fts3Phrase *pPhrase, /* Phrase to advance token of */ - int iToken, /* Specific token to advance */ - TokenDoclist *p, /* OUT: Docid and doclist for new entry */ - u8 *pbEof /* OUT: True if iterator is at EOF */ -){ - int rc = SQLITE_OK; - - if( pPhrase->iDoclistToken==iToken ){ - assert( p->bIgnore==0 ); - assert( pPhrase->aToken[iToken].pSegcsr==0 ); - fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); - p->pList = pPhrase->doclist.pList; - p->nList = pPhrase->doclist.nList; - p->iDocid = pPhrase->doclist.iDocid; - }else{ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - assert( pToken->pDeferred==0 ); - assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); - if( pToken->pSegcsr ){ - assert( p->bIgnore==0 ); - rc = sqlite3Fts3MsrIncrNext( - pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList - ); - if( p->pList==0 ) *pbEof = 1; - }else{ - p->bIgnore = 1; - } - } +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu); - return rc; -} +/* +** Close an RBU handle. +** +** If the RBU update has been completely applied, mark the RBU database +** as fully applied. Otherwise, assuming no error has occurred, save the +** current state of the RBU update appliation to the RBU database. +** +** If an error has already occurred as part of an sqlite3rbu_step() +** or sqlite3rbu_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, *pzErrmsg may be set to +** point to a buffer containing a utf-8 formatted English language error +** message. It is the responsibility of the caller to eventually free any +** such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current RBU update was started. +*/ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); /* -** The phrase iterator passed as the second argument: +** Create an RBU VFS named zName that accesses the underlying file-system +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** then the new RBU VFS uses the default system VFS to access the file-system. +** The new object is registered as a non-default VFS with SQLite before +** returning. ** -** * features at least one token that uses an incremental doclist, and +** Part of the RBU implementation uses a custom VFS object. Usually, this +** object is created and deleted automatically by RBU. ** -** * does not contain any deferred tokens. +** The exception is for applications that also use zipvfs. In this case, +** the custom VFS must be explicitly created by the user before the RBU +** handle is opened. The RBU VFS should be installed so that the zipvfs +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** (for example multiplexor) to access the file-system. For example, +** to assemble an RBU enabled VFS stack that uses both zipvfs and +** multiplexor (error checking omitted): ** -** Advance it to the next matching documnent in the database and populate -** the Fts3Doclist.pList and nList fields. +** // Create a VFS named "multiplex" (not the default). +** sqlite3_multiplex_initialize(0, 0); ** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. +** // Create an rbu VFS named "rbu" that uses multiplexor. If the +** // second argument were replaced with NULL, the "rbu" VFS would +** // access the file-system via the system default VFS, bypassing the +** // multiplexor. +** sqlite3rbu_create_vfs("rbu", "multiplex"); ** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** // Create a zipvfs VFS named "zipvfs" that uses rbu. +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** +** // Make zipvfs the default VFS. +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** +** Because the default VFS created above includes a RBU functionality, it +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack +** that does not include the RBU layer results in an error. +** +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality +** occasionally. */ -static int fts3EvalIncrPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - u8 bEof = 0; - - /* This is only called if it is guaranteed that the phrase has at least - ** one incremental token. In which case the bIncr flag is set. */ - assert( p->bIncr==1 ); - - if( p->nToken==1 && p->bIncr ){ - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &pDL->iDocid, &pDL->pList, &pDL->nList - ); - if( pDL->pList==0 ) bEof = 1; - }else{ - int bDescDoclist = pCsr->bDesc; - struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; - - memset(a, 0, sizeof(a)); - assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); - assert( p->iDoclistTokennToken && bEof==0; i++){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ - iMax = a[i].iDocid; - bMaxSet = 1; - } - } - assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); - assert( rc!=SQLITE_OK || bMaxSet ); +#if 0 +} /* end of the 'extern "C"' block */ +#endif - /* Keep advancing iterators until they all point to the same document */ - for(i=0; inToken; i++){ - while( rc==SQLITE_OK && bEof==0 - && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 - ){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ - iMax = a[i].iDocid; - i = 0; - } - } - } +#endif /* _SQLITE3RBU_H */ - /* Check if the current entries really are a phrase match */ - if( bEof==0 ){ - int nList = 0; - int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+1); - if( !aDoclist ) return SQLITE_NOMEM; - memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); +/************** End of sqlite3rbu.h ******************************************/ +/************** Continuing where we left off in sqlite3rbu.c *****************/ - for(i=0; i<(p->nToken-1); i++){ - if( a[i].bIgnore==0 ){ - char *pL = a[i].pList; - char *pR = aDoclist; - char *pOut = aDoclist; - int nDist = p->nToken-1-i; - int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); - if( res==0 ) break; - nList = (int)(pOut - aDoclist); - } - } - if( i==(p->nToken-1) ){ - pDL->iDocid = iMax; - pDL->pList = aDoclist; - pDL->nList = nList; - pDL->bFreeList = 1; - break; - } - sqlite3_free(aDoclist); - } - } - } +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif - *pbEof = bEof; - return rc; -} +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 /* -** Attempt to move the phrase iterator to point to the next matching docid. -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. -** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. +** Swap two objects of type TYPE. */ -static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - - if( p->bIncr ){ - rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ - sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, - &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof - ); - pDL->pList = pDL->pNextDocid; - }else{ - fts3EvalDlPhraseNext(pTab, pDL, pbEof); - } - - return rc; -} +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif /* -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, fts3EvalPhraseStart() is called on all phrases within the -** expression. Also the Fts3Expr.bDeferred variable is set to true for any -** expressions for which all descendent tokens are deferred. -** -** If parameter bOptOk is zero, then it is guaranteed that the -** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for -** each phrase in the expression (subject to deferred token processing). -** Or, if bOptOk is non-zero, then one or more tokens within the expression -** may be loaded incrementally, meaning doclist.aAll/nAll is not available. -** -** If an error occurs within this function, *pRc is set to an SQLite error -** code before returning. +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. */ -static void fts3EvalStartReaders( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expression to initialize phrases in */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; - int nToken = pExpr->pPhrase->nToken; - for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; - } - pExpr->bDeferred = (i==nToken); - *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); - }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); - pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); - } - } -} +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 /* -** An array of the following structures is assembled as part of the process -** of selecting tokens to defer before the query starts executing (as part -** of the xFilter() method). There is one element in the array for each -** token in the FTS expression. -** -** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong -** to phrases that are connected only by AND and NEAR operators (not OR or -** NOT). When determining tokens to defer, each AND/NEAR cluster is considered -** separately. The root of a tokens AND/NEAR cluster is stored in -** Fts3TokenAndCost.pRoot. +** A structure to store values read from the rbu_state table in memory. */ -typedef struct Fts3TokenAndCost Fts3TokenAndCost; -struct Fts3TokenAndCost { - Fts3Phrase *pPhrase; /* The phrase the token belongs to */ - int iToken; /* Position of token in phrase */ - Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ - int nOvfl; /* Number of overflow pages to load doclist */ - int iCol; /* The column the token must match */ +struct RbuState { + int eStage; + char *zTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; +}; + +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; }; /* -** This function is used to populate an allocated Fts3TokenAndCost array. +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: ** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if an error occurs during execution, *pRc is set to an -** SQLite error code. +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** */ -static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ - Fts3Expr *pExpr, /* Expression to consider */ - Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ - Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - for(i=0; *pRc==SQLITE_OK && inToken; i++){ - Fts3TokenAndCost *pTC = (*ppTC)++; - pTC->pPhrase = pPhrase; - pTC->iToken = i; - pTC->pRoot = pRoot; - pTC->pToken = &pPhrase->aToken[i]; - pTC->iCol = pPhrase->iColumn; - *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); - } - }else if( pExpr->eType!=FTSQUERY_NOT ){ - assert( pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND - || pExpr->eType==FTSQUERY_NEAR - ); - assert( pExpr->pLeft && pExpr->pRight ); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pLeft; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pRight; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); - } - } -} +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; /* -** Determine the average document (row) size in pages. If successful, -** write this value to *pnPage and return SQLITE_OK. Otherwise, return -** an SQLite error code. +** Values for RbuObjIter.eType ** -** The average document size in pages is calculated by first calculating -** determining the average size in bytes, B. If B is less than the amount -** of data that will fit on a single leaf page of an intkey table in -** this database, then the average docsize is 1. Otherwise, it is 1 plus -** the number of overflow pages consumed by a record B bytes in size. +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. */ -static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - int rc; - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anDoc = nDoc; - pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_IDX_DELETE 3 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 4 /* Insert on an aux. index b-tree */ +#define RBU_UPDATE 5 /* Update a row in a main table b-tree */ - *pnPage = pCsr->nRowAvg; - return SQLITE_OK; -} /* -** This function is called to select the tokens (if any) that will be -** deferred. The array aTC[] has already been populated when this is -** called. -** -** This function is called once for each AND/NEAR cluster in the -** expression. Each invocation determines which tokens to defer within -** the cluster with root node pRoot. See comments above the definition -** of struct Fts3TokenAndCost for more details. -** -** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() -** called on each token to defer. Otherwise, an SQLite error code is -** returned. +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. */ -static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Consider tokens with this root node */ - Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ - int nTC /* Number of entries in aTC[] */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int nDocSize = 0; /* Number of pages per doc loaded */ - int rc = SQLITE_OK; /* Return code */ - int ii; /* Iterator variable for various purposes */ - int nOvfl = 0; /* Total overflow pages used by doclists */ - int nToken = 0; /* Total number of tokens in cluster */ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; - int nMinEst = 0; /* The minimum count for any phrase so far. */ - int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ +/* +** RBU handle. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + i64 iOalSz; + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; +}; - /* Tokens are never deferred for FTS tables created using the content=xxx - ** option. The reason being that it is not guaranteed that the content - ** table actually contains the same data as the index. To prevent this from - ** causing any problems, the deferred token optimization is completely - ** disabled for content=xxx tables. */ - if( pTab->zContentTbl ){ - return SQLITE_OK; - } +/* +** An rbu VFS is implemented using an instance of this structure. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + rbu_file *pMain; /* Linked list of main db files */ +}; - /* Count the tokens in this AND/NEAR cluster. If none of the doclists - ** associated with the tokens spill onto overflow pages, or if there is - ** only 1 token, exit early. No tokens to defer in this case. */ - for(ii=0; ii0 ); + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ - /* Iterate through all tokens in this AND/NEAR cluster, in ascending order - ** of the number of overflow pages that will be loaded by the pager layer - ** to retrieve the entire doclist for the token from the full-text index. - ** Load the doclists for tokens that are either: - ** - ** a. The cheapest token in the entire query (i.e. the one visited by the - ** first iteration of this loop), or - ** - ** b. Part of a multi-token phrase. - ** - ** After each token doclist is loaded, merge it with the others from the - ** same phrase and count the number of documents that the merged doclist - ** contains. Set variable "nMinEst" to the smallest number of documents in - ** any phrase doclist for which 1 or more token doclists have been loaded. - ** Let nOther be the number of other phrases for which it is certain that - ** one or more tokens will not be deferred. - ** - ** Then, for each token, defer it if loading the doclist would result in - ** loading N or more overflow pages into memory, where N is computed as: - ** - ** (nMinEst + 4^nOther - 1) / (4^nOther) - */ - for(ii=0; iinOvfl) - ){ - pTC = &aTC[iTC]; - } - } - assert( pTC ); - if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ - /* The number of overflow pages to load for this (and therefore all - ** subsequent) tokens is greater than the estimated number of pages - ** that will be loaded if all subsequent tokens are deferred. - */ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - }else{ - /* Set nLoad4 to the value of (4^nOther) for the next iteration of the - ** for-loop. Except, limit the value to 2^24 to prevent it from - ** overflowing the 32-bit integer it is stored in. */ - if( ii<12 ) nLoad4 = nLoad4*4; +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ - if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ - /* Either this is the cheapest token in the entire query, or it is - ** part of a multi-token phrase. Either way, the entire doclist will - ** (eventually) be loaded into memory. It may as well be now. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - int nCount; - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); - nCount = fts3DoclistCountDocids( - pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll - ); - if( ii==0 || nCount=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; } } - pTC->pToken = 0; } + /* ERROR: unterminated delta */ + return -1; +} - return rc; +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; } /* -** This function is called from within the xFilter method. It initializes -** the full-text query currently stored in pCsr->pExpr. To iterate through -** the results of a query, the caller does: +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). ** -** fts3EvalStart(pCsr); -** while( 1 ){ -** fts3EvalNext(pCsr); -** if( pCsr->bEof ) break; -** ... return row pCsr->iPrevId to the caller ... -** } +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. */ -static int fts3EvalStart(Fts3Cursor *pCsr){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int nToken = 0; - int nOr = 0; - - /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; - /* Determine which, if any, tokens in the expression should be deferred. */ -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ - Fts3TokenAndCost *aTC; - Fts3Expr **apOr; - aTC = (Fts3TokenAndCost *)sqlite3_malloc( - sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr * 2 - ); - apOr = (Fts3Expr **)&aTC[nToken]; + int nOut; + int nOut2; + char *aOut; - if( !aTC ){ - rc = SQLITE_NOMEM; - }else{ - int ii; - Fts3TokenAndCost *pTC = aTC; - Fts3Expr **ppOr = apOr; + assert( argc==2 ); - fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = (int)(pTC-aTC); - nOr = (int)(ppOr-apOr); + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); - if( rc==SQLITE_OK ){ - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } return rc; } /* -** Invalidate the current position list for phrase pPhrase. +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). */ -static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ - if( pPhrase->doclist.bFreeList ){ - sqlite3_free(pPhrase->doclist.pList); +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); } - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; + return rc; } /* -** This function is called to edit the position list associated with -** the phrase object passed as the fifth argument according to a NEAR -** condition. For example: -** -** abc NEAR/5 "def ghi" -** -** Parameter nNear is passed the NEAR distance of the expression (5 in -** the example above). When this function is called, *paPoslist points to -** the position list, and *pnToken is the number of phrase tokens in, the -** phrase on the other side of the NEAR operator to pPhrase. For example, -** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to -** the position list associated with phrase "abc". +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. ** -** All positions in the pPhrase position list that are not sufficiently -** close to a position in the *paPoslist position list are removed. If this -** leaves 0 positions, zero is returned. Otherwise, non-zero. +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). ** -** Before returning, *paPoslist is set to point to the position lsit -** associated with pPhrase. And *pnToken is set to the number of tokens in -** pPhrase. +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. */ -static int fts3EvalNearTrim( - int nNear, /* NEAR distance. As in "NEAR/nNear". */ - char *aTmp, /* Temporary space to use */ - char **paPoslist, /* IN/OUT: Position list */ - int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ - Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql ){ - int nParam1 = nNear + pPhrase->nToken; - int nParam2 = nNear + *pnToken; - int nNew; - char *p2; - char *pOut; - int res; - - assert( pPhrase->doclist.pList ); + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + return rc; +} - p2 = pOut = pPhrase->doclist.pList; - res = fts3PoslistNearMerge( - &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 - ); - if( res ){ - nNew = (int)(pOut - pPhrase->doclist.pList) - 1; - assert( pPhrase->doclist.pList[nNew]=='\0' ); - assert( nNew<=pPhrase->doclist.nList && nNew>0 ); - memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); - pPhrase->doclist.nList = nNew; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; +/* +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). +*/ +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; } + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; +} - return res; +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is called. -** Otherwise, it advances the expression passed as the second argument to -** point to the next matching row in the database. Expressions iterate through -** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, -** or descending if it is non-zero. -** -** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if -** successful, the following variables in pExpr are set: -** -** Fts3Expr.bEof (non-zero if EOF - there is no next row) -** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) -** -** If the expression is of type FTSQUERY_PHRASE, and the expression is not -** at EOF, then the following variables are populated with the position list -** for the phrase for the visited row: -** -** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) -** FTs3Expr.pPhrase->doclist.pList (pointer to position list) -** -** It says above that this function advances the expression to the next -** matching row. This is usually true, but there are the following exceptions: -** -** 1. Deferred tokens are not taken into account. If a phrase consists -** entirely of deferred tokens, it is assumed to match every row in -** the db. In this case the position-list is not populated at all. -** -** Or, if a phrase contains one or more deferred tokens and one or -** more non-deferred tokens, then the expression is advanced to the -** next possible match, considering only non-deferred tokens. In other -** words, if the phrase is "A B C", and "B" is deferred, the expression -** is advanced to the next row that contains an instance of "A * C", -** where "*" may match any single token. The position list in this case -** is populated as for "A * C" before returning. +** Advance the iterator to the next position. ** -** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is -** advanced to point to the next row that matches "x AND y". -** -** See fts3EvalTestDeferredAndNear() for details on testing if a row is -** really a match, taking into account deferred tokens and NEAR operators. +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. */ -static void fts3EvalNextRow( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expr. to advance to next matching row */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } - if( pLeft->bDeferred ){ - /* LHS is entirely deferred. So we assume it matches every row. - ** Advance the RHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - /* RHS is entirely deferred. So we assume it matches every row. - ** Advance the LHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; }else{ - /* Neither the RHS or LHS are deferred. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else{ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNextRow(pCsr, pRight, pRc); - }else{ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } } - - break; } + } + } - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; + } + return rc; +} - if( pRight->bStart==0 ){ - fts3EvalNextRow(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - fts3EvalNextRow(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one argument - the name of a table in the RBU database. If the +** table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +*/ +static void rbuTargetNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zIn; + assert( argc==1 ); - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalInvalidatePoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); } } } /* -** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR -** cluster, then this function returns 1 immediately. +** Initialize the iterator structure passed as the second argument. ** -** Otherwise, it checks if the current row really does match the NEAR -** expression, using the data currently stored in the position lists -** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. -** -** If the current row is a match, the position list associated with each -** phrase in the NEAR expression is edited in place to contain only those -** phrase instances sufficiently close to their peers to satisfy all NEAR -** constraints. In this case it returns 1. If the NEAR expression does not -** match the current row, 0 is returned. The position lists may or may not -** be edited if 0 is returned. +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. */ -static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ - int res = 1; +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); - /* The following block runs if pExpr is the root of a NEAR query. - ** For example, the query: - ** - ** "w" NEAR "x" NEAR "y" NEAR "z" - ** - ** which is represented in tree form as: - ** - ** | - ** +--NEAR--+ <-- root of NEAR query - ** | | - ** +--NEAR--+ "z" - ** | | - ** +--NEAR--+ "y" - ** | | - ** "w" "x" - ** - ** The right-hand child of a NEAR node is always a phrase. The - ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this - it's the way the parser in fts3_expr.c works. - */ - if( *pRc==SQLITE_OK - && pExpr->eType==FTSQUERY_NEAR - && pExpr->bEof==0 - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - int nTmp = 0; /* Bytes of temp space */ - char *aTmp; /* Temp space for PoslistNearMerge() */ + rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + "ORDER BY name" + ); - /* Allocate temporary working space. */ - for(p=pExpr; p->pLeft; p=p->pLeft){ - nTmp += p->pRight->pPhrase->doclist.nList; - } - nTmp += p->pPhrase->doclist.nList; - if( nTmp==0 ){ - res = 0; - }else{ - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_master " + " WHERE type='index' AND tbl_name = ?" + ); + } - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - } +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} - sqlite3_free(aTmp); +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); } } - - return res; + sqlite3_free(zSql); + va_end(ap); + return p->rc; } /* -** This function is a helper function for fts3EvalTestDeferredAndNear(). -** Assuming no error occurs or has occurred, It returns non-zero if the -** expression passed as the second argument matches the row that pCsr -** currently points to, or zero if it does not. +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. ** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** If an error occurs during execution of this function, *pRc is set to -** the appropriate SQLite error code. In this case the returned value is -** undefined. +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. */ -static int fts3EvalTestExpr( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ - int *pRc /* IN/OUT: Error code */ -){ - int bHit = 1; /* Return value */ - if( *pRc==SQLITE_OK ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - && fts3EvalNearTest(pExpr, pRc) - ); +static void *rbuMalloc(sqlite3rbu *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} - /* If the NEAR expression does not match any rows, zero the doclist for - ** all phrases involved in the NEAR. This is because the snippet(), - ** offsets() and matchinfo() functions are not supposed to recognize - ** any instances of phrases that are part of unmatched NEAR queries. - ** For example if this expression: - ** - ** ... MATCH 'a OR (b NEAR c)' - ** - ** is matched against a row containing: - ** - ** 'a b d e' - ** - ** then any snippet() should ony highlight the "a" term, not the "b" - ** (as "b" is part of a non-matching NEAR clause). - */ - if( bHit==0 - && pExpr->eType==FTSQUERY_NEAR - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pRight->pPhrase); - } - } - if( p->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pPhrase); - } - } - break; +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; - case FTSQUERY_OR: { - int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); - bHit = bHit1 || bHit2; - break; - } + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } +} - case FTSQUERY_NOT: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - ); - break; +/* +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. +*/ +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; - default: { -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); - if( pExpr->bDeferred ){ - fts3EvalInvalidatePoslist(pPhrase); - } - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; - }else -#endif - { - bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); - } - break; - } + assert( *pRc==SQLITE_OK ); + if( zStr ){ + int nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; } } - return bHit; + + return zRet; } /* -** This function is called as the second part of each xNext operation when -** iterating through the results of a full-text query. At this point the -** cursor points to a row that matches the query expression, with the -** following caveats: -** -** * Up until this point, "NEAR" operators in the expression have been -** treated as "AND". -** -** * Deferred tokens have not yet been considered. -** -** If *pRc is not SQLITE_OK when this function is called, it immediately -** returns 0. Otherwise, it tests whether or not after considering NEAR -** operators and deferred tokens the current row is still a match for the -** expression. It returns 1 if both of the following are true: -** -** 1. *pRc is SQLITE_OK when this function returns, and -** -** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does *not* match the query. +** Finalize the statement passed as the second argument. ** -** Or, if no error occurs and it seems the current row does match the FTS -** query, return 0. +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. */ -static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ - int rc = *pRc; - int bMiss = 0; - if( rc==SQLITE_OK ){ +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_master ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_master ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_master where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%'), rootpage" + " FROM sqlite_master" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); - /* If there are one or more deferred tokens, load the current row into - ** memory and scan it to determine the position list for each deferred - ** token. Then, see if this row is really a match, considering deferred - ** tokens and NEAR operators (neither of which were taken into account - ** earlier, by fts3EvalNextRow()). - */ - if( pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } } + goto rbuTableType_end; } - bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + } - /* Free the position-lists accumulated for each deferred token above. */ - sqlite3Fts3FreeDeferredDoclists(pCsr); - *pRc = rc; + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } + +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. */ -static int fts3EvalNext(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Expr *pExpr = pCsr->pExpr; - assert( pCsr->isEof==0 ); - if( pExpr==0 ){ - pCsr->isEof = 1; - }else{ - do { - if( pCsr->isRequireSeek==0 ){ - sqlite3_reset(pCsr->pStmt); - } - assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNextRow(pCsr, pExpr, &rc); - pCsr->isEof = pExpr->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; + + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); } - /* Check if the cursor is past the end of the docid range specified - ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ - if( rc==SQLITE_OK && ( - (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) - || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) - )){ - pCsr->isEof = 1; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + } + rbuFinalize(p, pXInfo); + bIndex = 1; } - return rc; + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; } + /* -** Restart interation for expression pExpr so that the next call to -** fts3EvalNext() visits the first row. Do not allow incremental -** loading or merging of phrase doclists for this iteration. +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. ** -** If *pRc is other than SQLITE_OK when this function is called, it is -** a no-op. If an error occurs within this function, *pRc is set to an -** SQLite error code before returning. +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. */ -static void fts3EvalRestart( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc -){ - if( pExpr && *pRc==SQLITE_OK ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); - if( pPhrase ){ - fts3EvalInvalidatePoslist(pPhrase); - if( pPhrase->bIncr ){ - int i; - for(i=0; inToken; i++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[i]; - assert( pToken->pDeferred==0 ); - if( pToken->pSegcsr ){ - sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); - } - } - *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; } - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; } + sqlite3_finalize(pStmt); + pStmt = 0; - pExpr->iDocid = 0; - pExpr->bEof = 0; - pExpr->bStart = 0; - - fts3EvalRestart(pCsr, pExpr->pLeft, pRc); - fts3EvalRestart(pCsr, pExpr->pRight, pRc); - } -} + if( p->rc==SQLITE_OK + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } -/* -** After allocating the Fts3Expr.aMI[] array for each phrase in the -** expression rooted at pExpr, the cursor iterates through all rows matched -** by pExpr, calling this function for each row. This function increments -** the values in Fts3Expr.aMI[] according to the position-list currently -** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase -** expression nodes. -*/ -static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase && pPhrase->doclist.pList ){ - int iCol = 0; - char *p = pPhrase->doclist.pList; + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); - assert( *p ); - while( 1 ){ - u8 c = 0; - int iCnt = 0; - while( 0xFE & (*p | c) ){ - if( (c&0x80)==0 ) iCnt++; - c = *p++ & 0x80; + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); } - /* aMI[iCol*3 + 1] = Number of occurrences - ** aMI[iCol*3 + 2] = Number of rows containing at least one instance - */ - pExpr->aMI[iCol*3 + 1] += iCnt; - pExpr->aMI[iCol*3 + 2] += (iCnt>0); - if( *p==0x00 ) break; - p++; - p += fts3GetVarint32(p, &iCol); + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + pIter->abTblPk[iOrder] = (iPk!=0); + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; } } - fts3EvalUpdateCounts(pExpr->pLeft); - fts3EvalUpdateCounts(pExpr->pRight); + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); } + + return p->rc; } /* -** Expression pExpr must be of type FTSQUERY_PHRASE. +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. ** -** If it is not already allocated and populated, this function allocates and -** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part -** of a NEAR expression, then it also allocates and populates the same array -** for all other phrases that are part of the NEAR expression. +** If the index is of the following form: ** -** SQLITE_OK is returned if the aMI[] array is successfully allocated and -** populated. Otherwise, if an error occurs, an SQLite error code is returned. +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... */ -static int fts3EvalGatherStats( - Fts3Cursor *pCsr, /* Cursor object */ - Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ ){ - int rc = SQLITE_OK; /* Return code */ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ - assert( pExpr->eType==FTSQUERY_PHRASE ); - if( pExpr->aMI==0 ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } - sqlite3_int64 iPrevId = pCsr->iPrevId; - sqlite3_int64 iDocid; - u8 bEof; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + const char *zType; - /* Find the root of the NEAR expression */ - pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ - pRoot = pRoot->pParent; + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; } - iDocid = pRoot->iDocid; - bEof = pRoot->bEof; - assert( pRoot->bStart ); - /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol + ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; + } - fts3EvalRestart(pCsr, pRoot, &rc); - - while( pCsr->isEof==0 && rc==SQLITE_OK ){ + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; - do { - /* Ensure the %_content statement is reset. */ - if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); - assert( sqlite3_data_count(pCsr->pStmt)==0 ); + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } - /* Advance to the next document */ - fts3EvalNextRow(pCsr, pRoot, &rc); - pCsr->isEof = pRoot->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pRoot->iDocid; - }while( pCsr->isEof==0 - && pRoot->eType==FTSQUERY_NEAR - && fts3EvalTestDeferredAndNear(pCsr, &rc) - ); + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} - if( rc==SQLITE_OK && pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pRoot); +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; } } - pCsr->isEof = 0; - pCsr->iPrevId = iPrevId; - - if( bEof ){ - pRoot->bEof = bEof; - }else{ - /* Caution: pRoot may iterate through docids in ascending or descending - ** order. For this reason, even though it seems more defensive, the - ** do loop can not be written: - ** - ** do {...} while( pRoot->iDocidbEof==0 ); - }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalTestDeferredAndNear(pCsr, &rc); + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); } } - return rc; + return zList; } /* -** This function is used by the matchinfo() module to query a phrase -** expression node for the following information: -** -** 1. The total number of occurrences of the phrase in each column of -** the FTS table (considering all rows), and +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: ** -** 2. For each column, the number of rows in the table for which the -** column contains at least one instance of the phrase. -** -** If no error occurs, SQLITE_OK is returned and the values for each column -** written into the array aiOut as follows: -** -** aiOut[iCol*3 + 1] = Number of occurrences -** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); ** -** Caveats: -** -** * If a phrase consists entirely of deferred tokens, then all output -** values are set to the number of documents in the table. In other -** words we assume that very common tokens occur exactly once in each -** column of each row of the table. -** -** * If a phrase contains some deferred tokens (and some non-deferred -** tokens), count the potential occurrence identified by considering -** the non-deferred tokens instead of actual phrase occurrences. +** Return the string: ** -** * If the phrase is part of a NEAR expression, then only phrase instances -** that meet the NEAR constraint are included in the counts. +** "b = ?1 AND c = ?2" */ -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Phrase expression */ - u32 *aiOut /* Array to write results into (see above) */ +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int iCol; - - if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ - assert( pCsr->nDoc>0 ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; - aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); + }else{ - rc = fts3EvalGatherStats(pCsr, pExpr); - if( rc==SQLITE_OK ){ - assert( pExpr->aMI ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; - aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; } } } + return zList; +} - return rc; +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. +*/ +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); } + /* -** The expression pExpr passed as the second argument to this function -** must be of type FTSQUERY_PHRASE. -** -** The returned value is either NULL or a pointer to a buffer containing -** a position-list indicating the occurrences of the phrase in column iCol -** of the current row. -** -** More specifically, the returned buffer contains 1 varint for each -** occurrence of the phrase in the column, stored using the normal (delta+2) -** compression and is terminated by either an 0x01 or 0x00 byte. For example, -** if the requested column contains "a b X c d X X" and the position-list -** for 'X' is requested, the buffer returned may contain: +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. ** -** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). ** -** This function works regardless of whether or not the phrase is deferred, -** incremental, or neither. +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. */ -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol, /* Column to return position list for */ - char **ppOut /* OUT: Pointer to position list */ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter; - int iThis; - sqlite3_int64 iDocid; + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; - /* If this phrase is applies specifically to some column other than - ** column iCol, return a NULL pointer. */ - *ppOut = 0; - assert( iCol>=0 && iColnColumn ); - if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ - return SQLITE_OK; + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } } + return zList; +} - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; - if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ - int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ - int iMul; /* +1 if csr dir matches index dir, else -1 */ - int bOr = 0; - u8 bEof = 0; - u8 bTreeEof = 0; - Fts3Expr *p; /* Used to iterate from pExpr to root */ - Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ +/* +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + int nByte = nBind*2 + 1; - /* Check if this phrase descends from an OR expression node. If not, - ** return NULL. Otherwise, the entry that corresponds to docid - ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the - ** tree that the node is part of has been marked as EOF, but the node - ** itself is not EOF, then it may point to an earlier entry. */ - pNear = pExpr; - for(p=pExpr->pParent; p; p=p->pParent){ - if( p->eType==FTSQUERY_OR ) bOr = 1; - if( p->eType==FTSQUERY_NEAR ) pNear = p; - if( p->bEof ) bTreeEof = 1; + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; ibIncr ){ - int rc = SQLITE_OK; - int bEofSave = pExpr->bEof; - fts3EvalRestart(pCsr, pExpr, &rc); - while( rc==SQLITE_OK && !pExpr->bEof ){ - fts3EvalNextRow(pCsr, pExpr, &rc); - if( bEofSave==0 && pExpr->iDocid==iDocid ) break; +/* +** The iterator currently points to a table (not index) of type +** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY +** declaration for the corresponding imposter table. For example, +** if the iterator points to a table created as: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID +** +** this function returns: +** +** PRIMARY KEY("b", "a" DESC) +*/ +static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){ + char *z = 0; + assert( pIter->zIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; } - pIter = pPhrase->doclist.pList; - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( rc!=SQLITE_OK ) return rc; } - - iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1); - while( bTreeEof==1 - && pNear->bEof==0 - && (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0 - ){ - int rc = SQLITE_OK; - fts3EvalNextRow(pCsr, pExpr, &rc); - if( rc!=SQLITE_OK ) return rc; - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); + } + return z; +} - bEof = (pPhrase->doclist.nAll==0); - assert( bDescDoclist==0 || bDescDoclist==1 ); - assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); +/* +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. +** +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** +*/ +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ - if( bEof==0 ){ - if( pCsr->bDesc==bDescDoclist ){ - int dummy; - if( pNear->bEof ){ - /* This expression is already at EOF. So position it to point to the - ** last entry in the doclist at pPhrase->doclist.aAll[]. Variable - ** iDocid is already set for this entry, so all that is required is - ** to set pIter to point to the first byte of the last position-list - ** in the doclist. - ** - ** It would also be correct to set pIter and iDocid to zero. In - ** this case, the first call to sqltie3Fts4DoclistPrev() below - ** would also move the iterator to point to the last entry in the - ** doclist. However, this is expensive, as to do so it has to - ** iterate through the entire doclist from start to finish (since - ** it does not know the docid for the last entry). */ - pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1]; - fts3ReversePoslist(pPhrase->doclist.aAll, &pIter); - } - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - sqlite3Fts3DoclistPrev( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } - }else{ - if( pNear->bEof ){ - pIter = 0; - iDocid = 0; - } - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ - sqlite3Fts3DoclistNext( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &bEof - ); - } + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_master WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); } } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); - if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; - } - if( pIter==0 ) return SQLITE_OK; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); - if( *pIter==0x01 ){ - pIter++; - pIter += fts3GetVarint32(pIter, &iThis); - }else{ - iThis = 0; - } - while( iThisdbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); } +} - *ppOut = ((iCol==iThis)?pIter:0); - return SQLITE_OK; +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } } /* -** Free all components of the Fts3Phrase structure that were allocated by -** the eval module. Specifically, this means to free: +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: ** -** * the contents of pPhrase->doclist, and -** * any Fts3MultiSegReader objects held by phrase tokens. +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. */ -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - if( pPhrase ){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - fts3EvalInvalidatePoslist(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; - } +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); } } +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; -/* -** Return SQLITE_CORRUPT_VTAB. -*/ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ - return SQLITE_CORRUPT_VTAB; + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } } -#endif -#if !SQLITE_CORE /* -** Initialize API pointer table, if required. +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. */ -#ifdef _WIN32 -__declspec(dllexport) -#endif -SQLITE_API int sqlite3_fts3_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ ){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; -#endif + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } -/************** End of fts3.c ************************************************/ -/************** Begin file fts3_aux.c ****************************************/ -/* -** 2011 Jan 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -*/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + int nBind = 0; -/* #include */ -/* #include */ + assert( pIter->eType!=RBU_PK_VTAB ); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); + } -typedef struct Fts3auxTable Fts3auxTable; -typedef struct Fts3auxCursor Fts3auxCursor; + /* And to delete index entries */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } -struct Fts3auxTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - Fts3Table *pFts3Tab; -}; + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM '%q' " + "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "UNION ALL " + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "ORDER BY %s%s", + zCollist, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, zLimit + ); + } + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); + } -struct Fts3auxCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3MultiSegReader csr; /* Must be right after "base" */ - Fts3SegFilter filter; - char *zStop; - int nStop; /* Byte-length of string zStop */ - int iLangid; /* Language id to query */ - int isEof; /* True if cursor is at EOF */ - sqlite3_int64 iRowid; /* Current rowid */ + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } - int iCol; /* Current value of 'col' column */ - int nStat; /* Size of aStat[] array */ - struct Fts3auxColstats { - sqlite3_int64 nDoc; /* 'documents' values for current csr row */ - sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ - } *aStat; -}; + /* Create the DELETE statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } -/* -** Schema of the terms table. -*/ -#define FTS3_AUX_SCHEMA \ - "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + if( pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(2, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(2, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s, rbu_control%s FROM '%q'%s", + zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + pIter->zDataTbl, zLimit + ) + ); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} /* -** This function does all the work for both the xConnect and xCreate methods. -** These tables have no persistent representation of their own, so xConnect -** and xCreate are identical operations. +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. */ -static int fts3auxConnectMethod( - sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ ){ - char const *zDb; /* Name of database (e.g. "main") */ - char const *zFts3; /* Name of fts3 table */ - int nDb; /* Result of strlen(zDb) */ - int nFts3; /* Result of strlen(zFts3) */ - int nByte; /* Bytes of space to allocate here */ - int rc; /* value returned by declare_vtab() */ - Fts3auxTable *p; /* Virtual table object to return */ + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; - UNUSED_PARAMETER(pUnused); - - /* The user should invoke this in one of two forms: - ** - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); - */ - if( argc!=4 && argc!=5 ) goto bad_args; + /* In case an error occurs */ + *ppStmt = 0; - zDb = argv[1]; - nDb = (int)strlen(zDb); - if( argc==5 ){ - if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ - zDb = argv[3]; - nDb = (int)strlen(zDb); - zFts3 = argv[4]; - }else{ - goto bad_args; + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; }else{ - zFts3 = argv[3]; + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); } - nFts3 = (int)strlen(zFts3); - rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); - if( rc!=SQLITE_OK ) return rc; + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; - nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3auxTable *)sqlite3_malloc(nByte); - if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; - p->pFts3Tab = (Fts3Table *)&p[1]; - p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; - p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; - p->pFts3Tab->db = db; - p->pFts3Tab->nIndex = 1; + if( zSet ){ + const char *zPrefix = ""; - memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); - memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); - sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } - *ppVtab = (sqlite3_vtab *)p; - return SQLITE_OK; + return p->rc; +} - bad_args: - *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); - return SQLITE_ERROR; +static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } + } + return db; } /* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. */ -static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3auxTable *p = (Fts3auxTable *)pVtab; - Fts3Table *pFts3 = p->pFts3Tab; - int i; +static void rbuOpenDatabase(sqlite3rbu *p){ + assert( p->rc==SQLITE_OK ); + assert( p->dbMain==0 && p->dbRbu==0 ); - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(pFts3->aStmt[i]); - } - sqlite3_free(pFts3->zSegmentsTbl); - sqlite3_free(p); - return SQLITE_OK; -} + p->eStage = 0; + p->dbMain = rbuOpenDbhandle(p, p->zTarget); + p->dbRbu = rbuOpenDbhandle(p, p->zRbu); -#define FTS4AUX_EQ_CONSTRAINT 1 -#define FTS4AUX_GE_CONSTRAINT 2 -#define FTS4AUX_LE_CONSTRAINT 4 + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); + } -/* -** xBestIndex - Analyze a WHERE and ORDER BY clause. -*/ -static int fts3auxBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo -){ - int i; - int iEq = -1; - int iGe = -1; - int iLe = -1; - int iLangid = -1; - int iNext = 1; /* Next free argvIndex value */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); + } - UNUSED_PARAMETER(pVTab); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } - /* This vtab delivers always results in "ORDER BY term ASC" order. */ - if( pInfo->nOrderBy==1 - && pInfo->aOrderBy[0].iColumn==0 - && pInfo->aOrderBy[0].desc==0 - ){ - pInfo->orderByConsumed = 1; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); } - /* Search for equality and range constraints on the "term" column. - ** And equality constraints on the hidden "languageid" column. */ - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable ){ - int op = pInfo->aConstraint[i].op; - int iCol = pInfo->aConstraint[i].iColumn; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); - if( iCol==0 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; - } - if( iCol==4 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; - } - } + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } - if( iEq>=0 ){ - pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = iNext++; - pInfo->estimatedCost = 5; - }else{ - pInfo->idxNum = 0; - pInfo->estimatedCost = 20000; - if( iGe>=0 ){ - pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } - if( iLe>=0 ){ - pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); } - if( iLangid>=0 ){ - pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; - pInfo->estimatedCost--; +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); } +#endif +} - return SQLITE_OK; +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; } /* -** xOpen - Open a cursor. +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. */ -static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ - UNUSED_PARAMETER(pVTab); + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); + } + } - pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); - if( !pCsr ) return SQLITE_NOMEM; - memset(pCsr, 0, sizeof(Fts3auxCursor)); + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } - *ppCsr = (sqlite3_vtab_cursor *)pCsr; - return SQLITE_OK; + if( p->rc==SQLITE_OK ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + + if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + } } /* -** xClose - Close a cursor. +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. */ -static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->zStop); - sqlite3_free(pCsr->aStat); - sqlite3_free(pCsr); - return SQLITE_OK; -} + if( pRbu->mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_INTERNAL; + } -static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ - if( nSize>pCsr->nStat ){ - struct Fts3auxColstats *aNew; - aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, - sizeof(struct Fts3auxColstats) * nSize - ); + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); if( aNew==0 ) return SQLITE_NOMEM; - memset(&aNew[pCsr->nStat], 0, - sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) - ); - pCsr->aStat = aNew; - pCsr->nStat = nSize; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame