home *** CD-ROM | disk | FTP | other *** search
/ Simtel MSDOS 1992 June / SIMTEL_0692.cdr / msdos / pgmutl / val_link.arc / TOKEN.C < prev    next >
Text File  |  1989-02-18  |  25KB  |  685 lines
  1. /*                                 TOKEN.C                                 */
  2.  
  3. /*+-------------------------------------------------------------------------+
  4.   |                                                                         |
  5.   |                       complete_a_filename_token                         |
  6.   |                                                                         |
  7.   +-------------------------------------------------------------------------+*/
  8. void complete_a_filename_token()
  9. BeginDeclarations
  10. EndDeclarations
  11. BeginCode
  12.  copy_string(token, next_token);
  13.  scan_out_token();
  14.  If token_type Is switch_end_token_type
  15.   Then
  16.    concat_string(token, next_token);
  17.    scan_out_token();
  18.    If token_type Is filename_token_type
  19.     Then
  20.      concat_string(token, next_token);
  21.      scan_out_token();
  22.     EndIf;
  23.   EndIf;
  24.  return;
  25. EndCode
  26.  
  27. /*+-------------------------------------------------------------------------+
  28.   |                                                                         |
  29.   |                          eat_white_space                                |
  30.   |                                                                         |
  31.   +-------------------------------------------------------------------------+*/
  32. void eat_white_space()
  33. BeginDeclarations
  34. EndDeclarations
  35. BeginCode
  36.  While token_break_char Is ' '
  37.   BeginWhile
  38.    token_get_char();
  39.   EndWhile;
  40. EndCode
  41.  
  42. /*+-------------------------------------------------------------------------+
  43.   |                                                                         |
  44.   |                     get_free_token_source_element                       |
  45.   |                                                                         |
  46.   +-------------------------------------------------------------------------+*/
  47. token_stack_ptr get_free_token_source_element()
  48. BeginDeclarations
  49. token_stack_ptr                        elem;
  50. EndDeclarations
  51. BeginCode
  52.  Pop token_stack_free_list InTo elem EndPop;
  53.  If elem IsNull
  54.   Then
  55.    elem = (token_stack_ptr) 
  56.            allocate_memory(Addr(static_pool),
  57.                            Bit_32(sizeof(token_stack_type)));
  58.   EndIf;
  59.  return(elem);
  60. EndCode
  61.  
  62. /*+-------------------------------------------------------------------------+
  63.   |                                                                         |
  64.   |                        get_filename_token                               |
  65.   |                                                                         |
  66.   +-------------------------------------------------------------------------+*/
  67. void get_filename_token(bit_16            required,
  68.                         file_info_list   *list)
  69. BeginDeclarations
  70. #define List                           (*list)
  71. token_stack_ptr                        source_element;
  72. #define Source_element                 (*source_element)
  73. EndDeclarations
  74. BeginCode
  75.  Loop
  76.   BeginLoop
  77.    required = (required) AndIf (List.first IsNull);
  78.    If token_type Is text_token_type
  79.     Then
  80.      linker_error(8, "Input syntax error:  \"%Fs\" out of place.\n",
  81.                      String(next_token));
  82.     EndIf;
  83.    If token_type Is indirect_file_token_type
  84.     Then
  85.      scan_out_token();
  86.      If token_type IsNot filename_token_type
  87.       Then
  88.        linker_error(8,"Input syntax error:  Expected filename after '@'.\n");
  89.       EndIf;
  90.      complete_a_filename_token();
  91.      source_element                    = get_free_token_source_element();
  92.      Source_element.break_char         = token_break_char;
  93.      Source_element.source_file        = input_open(token);
  94.      Source_element.token_string       = Null;
  95.      Source_element.token_string_index = 0;
  96.      Push source_element OnTo token_stack EndPush;
  97.      token_break_char                  = ' ';
  98.      scan_out_token();
  99.      ContinueLoop;
  100.     EndIf;
  101.    If token_type Is switch_token_type
  102.     Then
  103.      process_switch();
  104.      ContinueLoop;
  105.     EndIf;
  106.    If token_type Is continuation_token_type
  107.     Then
  108.      scan_out_token();
  109.      If token_type Is line_end_token_type
  110.       Then
  111.        scan_out_token();
  112.       EndIf;
  113.      ContinueLoop;
  114.     EndIf;
  115.    If token_type Is filename_token_type
  116.     Then
  117.      complete_a_filename_token();
  118.      more_tokens = True;
  119.      return;
  120.     EndIf;
  121.    If ((token_type Is end_of_command_line_token_type)  AndIf
  122.        (List.first IsNull))                            OrIf
  123.       ((required)                                      AndIf
  124.        ((token_type Is separator_token_type)           OrIf
  125.         (token_type Is line_end_token_type)))
  126.     Then
  127.      If (*token_stack.first).source_file IsNot stdin
  128.       Then
  129.        source_element                    = get_free_token_source_element();
  130.        Source_element.source_file        = stdin;
  131.        Source_element.token_string       = Null;
  132.        Source_element.token_string_index = 0;
  133.        Push source_element OnTo token_stack EndPush;
  134.        token_break_char                  = ' ';
  135.       EndIf;
  136.      If default_prompt IsNotNull
  137.       Then
  138.        linker_message(default_prompt, String(default_filename));
  139.        prompt_next_stdin = False;
  140.        default_prompt    = Null;
  141.       EndIf;
  142.      scan_out_token();
  143.      ContinueLoop;
  144.     EndIf;
  145.    If List.first IsNull
  146.     Then
  147.      If (token_type Is separator_token_type)  OrIf
  148.         (token_type Is terminator_token_type)
  149.       Then
  150.        If prompting_for Is 2
  151.         Then
  152.          If comfile.val IsTrue
  153.           Then
  154.            default_extension = com_extension_string;
  155.           Else
  156.            If sysfile.val IsTrue
  157.             Then
  158.              default_extension = sys_extension_string;
  159.             Else
  160.              default_extension = exe_extension_string;
  161.             EndIf;
  162.            EndIf;
  163.          change_extension(default_filename, default_extension);
  164.         EndIf;
  165.        copy_string(token, default_filename);
  166.        more_tokens = False;
  167.        return;
  168.       EndIf;
  169.     Else
  170.      If (token_type Is separator_token_type)           OrIf
  171.         (token_type Is terminator_token_type)          OrIf
  172.         (token_type Is end_of_command_line_token_type)
  173.       Then
  174.        copy_string(token, null_string);
  175.        more_tokens = False;
  176.        return;
  177.      EndIf;
  178.     EndIf;
  179.    If token_type Is line_end_token_type
  180.     Then
  181.      If List.first IsNull
  182.       Then
  183.        If prompting_for Is 2
  184.         Then
  185.          If comfile.val IsTrue
  186.           Then
  187.            default_extension = com_extension_string;
  188.           Else
  189.            If sysfile.val IsTrue
  190.             Then
  191.              default_extension = sys_extension_string;
  192.             Else
  193.              default_extension = exe_extension_string;
  194.             EndIf;
  195.            EndIf;
  196.          change_extension(default_filename, default_extension);
  197.         EndIf;
  198.        copy_string(token, default_filename);
  199.       Else
  200.        copy_string(token, null_string);
  201.       EndIf;
  202.      If (*token_stack.first).source_file Is stdin
  203.       Then
  204.        Pop token_stack InTo source_element EndPop;
  205.        Push source_element OnTo token_stack_free_list EndPush;
  206.       EndIf;
  207.      prompt_next_stdin = False;
  208.      more_tokens = False;
  209.      return;
  210.     EndIf;
  211.   EndLoop;
  212.  return;
  213. EndCode
  214. #undef List
  215. #undef Source_element
  216.  
  217. /*+-------------------------------------------------------------------------+
  218.   |                                                                         |
  219.   |                             input_open                                  |
  220.   |                                                                         |
  221.   +-------------------------------------------------------------------------+*/
  222. FILE *input_open(string_ptr fn)
  223. BeginDeclarations
  224. FILE                                  *infile;
  225. EndDeclarations
  226. BeginCode
  227.  infile = fopen((char *)near_string(fn), "r");
  228.  If infile Is NULL
  229.   Then
  230.    linker_error(8, "Could not open file \"%Fs\" for input.\n",
  231.                    String(fn));
  232.   EndIf;
  233.  return(infile);
  234. EndCode
  235.  
  236. /*+-------------------------------------------------------------------------+
  237.   |                                                                         |
  238.   |                             process_switch                              |
  239.   |                                                                         |
  240.   +-------------------------------------------------------------------------+*/
  241. void process_switch()
  242. BeginDeclarations
  243. switch_table_ptr                       current_switch;
  244. #define Current_switch                 (*current_switch)
  245. EndDeclarations
  246. BeginCode
  247.  current_switch = switch_table;
  248.  scan_out_token();
  249.  copy_string(token, next_token);
  250.  lowercase_string(token);
  251.  While Current_switch.full_name IsNotNull
  252.   BeginWhile
  253.    If compare_string(token, string((byte *)Current_switch.abbr_name)) IsZero
  254.     Then
  255.      Current_switch.switch_processor(current_switch);
  256.      return;
  257.     EndIf;
  258.    If (Length(token) NotLessThan Current_switch.min_length) AndIf
  259.       (far_compare(String(token), (byte *) Current_switch.full_name,
  260.         Length(token)) IsZero)
  261.     Then
  262.      Current_switch.switch_processor(current_switch);
  263.      return;
  264.     EndIf;
  265.    current_switch++;
  266.   EndWhile;
  267.  linker_error(8,"Syntax error:  \"%Fs\" is an unknown switch.\n",
  268.                 String(token));
  269.  return;
  270. EndCode
  271. #undef Current_switch
  272.  
  273. /*+-------------------------------------------------------------------------+
  274.   |                                                                         |
  275.   |                           scan_bit_16_switch                            |
  276.   |                                                                         |
  277.   +-------------------------------------------------------------------------+*/
  278. void scan_bit_16_switch(switch_table_ptr current_switch)
  279. BeginDeclarations
  280. #define Current_switch                 (*current_switch)
  281. #define Affected_thing (*((bit_16_switch_ptr)(*current_switch).affected_thing))
  282. EndDeclarations
  283. BeginCode
  284.  scan_out_token();
  285.  copy_string(token, next_token);
  286.  If token_type IsNot switch_end_token_type
  287.   Then
  288.    linker_error(8,"Syntax error:  \":\" did not follow switch \"%s\"\n",
  289.                   Current_switch.full_name);
  290.   EndIf;
  291.  scan_out_token();
  292.  copy_string(token, next_token);
  293.   If (Not token_is_number)                                OrIf
  294.      (token_numeric_value LessThan    Affected_thing.min) OrIf
  295.      (token_numeric_value GreaterThan Affected_thing.max)
  296.    Then
  297.    linker_error(8,"Syntax error:  Switch \"%s\" requires a numeric value\n"
  298.                   "               between %u and %u\n",
  299.                   Current_switch.full_name,
  300.                   Affected_thing.min, Affected_thing.max);
  301.    Else
  302.     Affected_thing.val = token_numeric_value;
  303.     Affected_thing.set = True;
  304.    EndIf;
  305.  scan_out_token();
  306.  copy_string(token, next_token);
  307.  return;
  308. EndCode
  309. #undef Current_switch
  310. #undef Affected_thing
  311.  
  312. /*+-------------------------------------------------------------------------+
  313.   |                                                                         |
  314.   |                             scan_opt_bit_16                             |
  315.   |                                                                         |
  316.   +-------------------------------------------------------------------------+*/
  317. void scan_opt_bit_16(switch_table_ptr current_switch)
  318. BeginDeclarations
  319. #define Current_switch                 (*current_switch)
  320. #define Affected_thing (*((bit_16_switch_ptr)(*current_switch).affected_thing))
  321. EndDeclarations
  322. BeginCode
  323.  scan_out_token();
  324.  copy_string(token, next_token);
  325.  Affected_thing.set = True;
  326.  If token_type IsNot switch_end_token_type
  327.   Then
  328.    Affected_thing.val = Affected_thing.def;
  329.    return;
  330.   EndIf;
  331.  scan_out_token();
  332.  copy_string(token, next_token);
  333.   If (Not token_is_number)                                OrIf
  334.      (token_numeric_value LessThan    Affected_thing.min) OrIf
  335.      (token_numeric_value GreaterThan Affected_thing.max)
  336.    Then
  337.    linker_error(8,"Syntax error:  Switch \"%s\" requires a numeric value\n"
  338.                   "               between %u and %u\n",
  339.                   Current_switch.full_name,
  340.                   Affected_thing.min, Affected_thing.max);
  341.    Else
  342.     Affected_thing.val = token_numeric_value;
  343.    EndIf;
  344.  scan_out_token();
  345.  copy_string(token, next_token);
  346.  return;
  347. EndCode
  348. #undef Current_switch
  349. #undef Affected_thing
  350.  
  351. /*+-------------------------------------------------------------------------+
  352.   |                                                                         |
  353.   |                         scan_out_token                                  |
  354.   |                                                                         |
  355.   +-------------------------------------------------------------------------+*/
  356. void scan_out_token()
  357. BeginDeclarations
  358. bit_16                                 paren_count;
  359. EndDeclarations
  360. BeginCode
  361.  eat_white_space();
  362.  copy_string(next_token, null_string);
  363.  token_is_hex_number =
  364.  token_is_number     = False;
  365.  Using token_break_char
  366.   BeginCase
  367.    When '\n':
  368.     prompt_next_stdin = True;
  369.     concat_char_to_string(next_token, token_break_char);
  370.     If token_stack.first Is token_stack.last
  371.      Then
  372.       token_type       = end_of_command_line_token_type;
  373.      Else
  374.       token_type       = line_end_token_type;
  375.      EndIf;
  376.     token_break_char = ' ';  /* Make it look like we advanced a character */
  377.     break;
  378.    When ',':
  379.     concat_char_to_string(next_token, token_break_char);
  380.     token_type       = separator_token_type;
  381.     token_break_char = ' ';  /* Make it look like we advanced a character */
  382.     break;
  383.    When ';':
  384.     concat_char_to_string(next_token, token_break_char);
  385.     token_type       = terminator_token_type;
  386.     break;
  387.    When '+':
  388.     concat_char_to_string(next_token, token_break_char);
  389.     token_type       = continuation_token_type;
  390.     token_break_char = ' ';  /* Make it look like we advanced a character */
  391.     break;
  392.    When '/':
  393.     concat_char_to_string(next_token, token_break_char);
  394.     token_type       = switch_token_type;
  395.     token_break_char = ' ';  /* Make it look like we advanced a character */
  396.     break;
  397.    When ':':
  398.     concat_char_to_string(next_token, token_break_char);
  399.     token_type       = switch_end_token_type;
  400.     token_break_char = ' ';  /* Make it look like we advanced a character */
  401.     break;
  402.    When '@':
  403.     concat_char_to_string(next_token, token_break_char);
  404.     token_type       = indirect_file_token_type;
  405.     token_break_char = ' ';  /* Make it look like we advanced a character */
  406.     break;
  407.    When '(':
  408.     paren_count = 1;
  409.     token_type  = text_token_type;
  410.     concat_char_to_string(next_token, token_break_char);
  411.     While paren_count IsNotZero
  412.      BeginWhile
  413.       token_get_char();
  414.       If token_break_char IsNot '\n'
  415.        Then
  416.         concat_char_to_string(next_token, token_break_char);
  417.        Else
  418.         If (*token_stack.first).source_file Is stdin
  419.          Then
  420.           linker_message("continue parenthesized text:  ");
  421.          EndIf;
  422.        EndIf;
  423.       If token_break_char Is '('
  424.        Then
  425.         paren_count++;
  426.        EndIf;
  427.       If token_break_char Is ')'
  428.        Then
  429.         paren_count--;
  430.        EndIf;
  431.      EndWhile;
  432.     token_break_char = ' ';  /* Make it look like we advanced a character */
  433.     break;
  434.    Otherwise:
  435.     token_is_number     = True;
  436.     token_numeric_value = 0;
  437.     While (token_break_char IsNot ',')  AndIf
  438.           (token_break_char IsNot ';')  AndIf
  439.           (token_break_char IsNot '+')  AndIf
  440.           (token_break_char IsNot '/')  AndIf
  441.           (token_break_char IsNot '@')  AndIf
  442.           (token_break_char IsNot ':')  AndIf
  443.           (token_break_char IsNot ' ')  AndIf
  444.           (token_break_char IsNot '\n')
  445.      BeginWhile
  446.       concat_char_to_string(next_token, token_break_char);
  447.       If (Length(next_token) Is 2) AndIf (String(next_token)[0] Is '0') AndIf
  448.          ((String(next_token)[1] Is 'x') OrIf (String(next_token)[1] Is 'X'))
  449.        Then
  450.         token_is_hex_number = True;
  451.        Else
  452.         If token_is_hex_number IsFalse
  453.          Then
  454.           token_is_number = token_is_number AndIf
  455.                             isdigit(token_break_char);
  456.           If token_is_number
  457.            Then
  458.             token_numeric_value = (token_numeric_value * 10) +
  459.                                   Bit_16(token_break_char - '0');
  460.            EndIf;
  461.          Else
  462.           token_is_hex_number =
  463.           token_is_number     = token_is_number AndIf
  464.                                 isxdigit(token_break_char);
  465.           If token_is_number
  466.            Then
  467.             If isdigit(token_break_char)
  468.              Then
  469.               token_numeric_value = (token_numeric_value * 16) +
  470.                                     Bit_16(token_break_char - '0');
  471.              Else
  472.               token_numeric_value = (token_numeric_value * 16) +
  473.                                  Bit_16(toupper(token_break_char) - 'A' + 10);
  474.              EndIf;
  475.            EndIf;
  476.          EndIf;
  477.        EndIf;
  478.       token_get_char();
  479.      EndWhile;
  480.     token_type = filename_token_type;
  481.     break;
  482.   EndCase;
  483.  return;
  484. EndCode
  485.  
  486. /*+-------------------------------------------------------------------------+
  487.   |                                                                         |
  488.   |                           scan_help_switch                              |
  489.   |                                                                         |
  490.   +-------------------------------------------------------------------------+*/
  491. void scan_help_switch(switch_table_ptr current_switch)
  492. BeginDeclarations
  493. #define Affected_thing (*((boolean_switch_ptr)(*current_switch).affected_thing))
  494. FILE                                  *help_file;
  495. EndDeclarations
  496. BeginCode
  497.  Affected_thing.val = True;
  498.  help_file = fopen(CharPtr(near_string(help_filename)), "r");
  499.  If help_file IsNull
  500.   Then
  501.    printf("Could not open help file \"%Fs\".\n", String(help_filename));
  502.   Else
  503.    While fgets(CharPtr(object_file_element), MAX_ELEMENT_SIZE, help_file) 
  504.          IsNotNull
  505.     BeginWhile
  506.      fputs(CharPtr(object_file_element), stdout);
  507.      If strcmp(CharPtr(object_file_element), 
  508.                "Press [RETURN] to continue.\n") IsZero
  509.       Then
  510.        gets(CharPtr(object_file_element));
  511.       EndIf;
  512.     EndWhile;
  513.    fclose(help_file);
  514.   EndIf;
  515.  exit(0);
  516.  return;
  517. EndCode
  518. #undef Affected_thing
  519.  
  520. /*+-------------------------------------------------------------------------+
  521.   |                                                                         |
  522.   |                           scan_reset_bit_16                             |
  523.   |                                                                         |
  524.   +-------------------------------------------------------------------------+*/
  525. void scan_reset_bit_16(switch_table_ptr current_switch)
  526. BeginDeclarations
  527. #define Affected_thing (*((bit_16_switch_ptr)(*current_switch).affected_thing))
  528. EndDeclarations
  529. BeginCode
  530.  scan_out_token();
  531.  copy_string(token, next_token);
  532.  Affected_thing.set = False;
  533.  Affected_thing.val = Affected_thing.def;
  534.  return;
  535. EndCode
  536. #undef Affected_thing
  537.  
  538. /*+-------------------------------------------------------------------------+
  539.   |                                                                         |
  540.   |                           scan_reset_switch                             |
  541.   |                                                                         |
  542.   +-------------------------------------------------------------------------+*/
  543. void scan_reset_switch(switch_table_ptr current_switch)
  544. BeginDeclarations
  545. #define Current_switch                 (*current_switch)
  546. #define Affected_thing (*((boolean_switch_ptr)(*current_switch).affected_thing))
  547. EndDeclarations
  548. BeginCode
  549.  Affected_thing.val = False;
  550.  scan_out_token();
  551.  copy_string(token, next_token);
  552.  return;
  553. EndCode
  554. #undef Current_switch
  555. #undef Affected_thing
  556.  
  557. /*+-------------------------------------------------------------------------+
  558.   |                                                                         |
  559.   |                            scan_set_switch                              |
  560.   |                                                                         |
  561.   +-------------------------------------------------------------------------+*/
  562. void scan_set_switch(switch_table_ptr current_switch)
  563. BeginDeclarations
  564. #define Current_switch                 (*current_switch)
  565. #define Affected_thing (*((boolean_switch_ptr)(*current_switch).affected_thing))
  566. EndDeclarations
  567. BeginCode
  568.  Affected_thing.val = True;
  569.  scan_out_token();
  570.  copy_string(token, next_token);
  571.  return;
  572. EndCode
  573. #undef Current_switch
  574. #undef Affected_thing
  575.  
  576. /*+-------------------------------------------------------------------------+
  577.   |                                                                         |
  578.   |                            scan_text_switch                             |
  579.   |                                                                         |
  580.   +-------------------------------------------------------------------------+*/
  581. void scan_text_switch(switch_table_ptr current_switch)
  582. BeginDeclarations
  583. #define Current_switch                 (*current_switch)
  584. #define Affected_thing (*((text_switch_ptr)(*current_switch).affected_thing))
  585. EndDeclarations
  586. BeginCode
  587.  scan_out_token();
  588.  copy_string(token, next_token);
  589.  If token_type IsNot switch_end_token_type
  590.   Then
  591.    linker_error(8,"Syntax error:  \":\" did not follow switch \"%s\"\n",
  592.                   Current_switch.full_name);
  593.   EndIf;
  594.  scan_out_token();
  595.  copy_string(token, next_token);
  596.  If token_type IsNot text_token_type
  597.   Then
  598.    linker_error(8, "Syntax error:  Parenthesized text did not follow\n"
  599.                    "\t\"%s\" switch.  Instead found \"%Fs\".\n",
  600.                   Current_switch.full_name, String(token));
  601.   EndIf;
  602.  Affected_thing.val = duplicate_string(Addr(static_pool), next_token);
  603.  scan_out_token();
  604.  copy_string(token, next_token);
  605.  return;
  606. EndCode
  607. #undef Affected_thing
  608. #undef Current_switch
  609.  
  610. /*+-------------------------------------------------------------------------+
  611.   |                                                                         |
  612.   |                           token_get_char                                |
  613.   |                                                                         |
  614.   +-------------------------------------------------------------------------+*/
  615. void token_get_char()
  616. BeginDeclarations
  617. int_16                                 c;
  618. token_stack_ptr                        tos;
  619. #define Tos                            (*tos)
  620. #define Tos_string                     (*tos).token_string
  621. EndDeclarations
  622. BeginCode
  623.  Loop
  624.   BeginLoop
  625.    If ((*token_stack.first).source_file Is stdin) AndIf
  626.       (prompt_next_stdin)
  627.     Then
  628.      linker_message("continue:  ");
  629.      prompt_next_stdin = False;
  630.     EndIf;
  631.    tos = token_stack.first;
  632.    If tos IsNull
  633.     Then
  634.      token_break_char = ';';
  635.      return;
  636.     EndIf;
  637.    If Tos.source_file IsNull
  638.     Then /* Input is from a string */
  639.      If Tos.token_string_index LessThan Length(Tos_string)
  640.       Then
  641.        token_break_char = String(Tos_string)[Tos.token_string_index++];
  642.        ExitLoop;
  643.       Else
  644.        If token_stack.first Is token_stack.last
  645.         Then
  646.          token_break_char = '\n';
  647.          return;
  648.         Else
  649.          Pop token_stack InTo tos EndPop;
  650.          Push tos OnTo token_stack_free_list EndPush;
  651.          ContinueLoop;
  652.         EndIf;
  653.       EndIf;
  654.     Else /* Input is from a file */
  655.      c = fgetc(Tos.source_file);
  656.      If c Is EOF
  657.       Then
  658.        If Tos.source_file IsNot stdin
  659.         Then
  660.          fclose(Tos.source_file);
  661.         EndIf;
  662.        Pop token_stack InTo tos EndPop;
  663.        token_break_char = Tos.break_char;
  664.        Push tos OnTo token_stack_free_list EndPush;
  665.        ExitLoop;
  666.       Else
  667.        token_break_char = Byte(c);
  668.        ExitLoop;
  669.       EndIf;
  670.     EndIf;
  671.   EndLoop;
  672.  If token_break_char Is '\r'
  673.   Then
  674.    token_break_char = '\n';
  675.   EndIf;
  676.  If token_break_char Is '\t'
  677.   Then
  678.    token_break_char = ' ';
  679.   EndIf;
  680.  return;
  681. EndCode
  682. #undef Tos
  683. #undef Tos_string
  684.  
  685.